1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVConstantPoolValue.h"
18#include "RISCVMachineFunctionInfo.h"
19#include "RISCVRegisterInfo.h"
20#include "RISCVSelectionDAGInfo.h"
21#include "RISCVSubtarget.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Analysis/MemoryLocation.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineJumpTableInfo.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/SDPatternMatch.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/IntrinsicsRISCV.h"
44#include "llvm/MC/MCCodeEmitter.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/InstructionCost.h"
50#include "llvm/Support/KnownBits.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/raw_ostream.h"
53#include <optional>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "riscv-lower"
58
59STATISTIC(NumTailCalls, "Number of tail calls");
60
61static cl::opt<unsigned> ExtensionMaxWebSize(
62 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
63 cl::desc("Give the maximum size (in number of nodes) of the web of "
64 "instructions that we will consider for VW expansion"),
65 cl::init(Val: 18));
66
67static cl::opt<bool>
68 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
69 cl::desc("Allow the formation of VW_W operations (e.g., "
70 "VWADD_W) with splat constants"),
71 cl::init(Val: false));
72
73static cl::opt<unsigned> NumRepeatedDivisors(
74 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
75 cl::desc("Set the minimum number of repetitions of a divisor to allow "
76 "transformation to multiplications by the reciprocal"),
77 cl::init(Val: 2));
78
79static cl::opt<int>
80 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
81 cl::desc("Give the maximum number of instructions that we will "
82 "use for creating a floating-point immediate value"),
83 cl::init(Val: 3));
84
85static cl::opt<bool>
86 ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
87 cl::desc("Swap add and addi in cases where the add may "
88 "be combined with a shift"),
89 cl::init(Val: true));
90
91static cl::opt<int> BrMergingBaseCostThresh(
92 "riscv-br-merging-base-cost", cl::init(Val: 2),
93 cl::desc(
94 "Sets the cost threshold for when multiple conditionals will be merged "
95 "into one branch versus be split in multiple branches. Merging "
96 "conditionals saves branches at the cost of additional instructions. "
97 "This value sets the instruction cost limit, below which conditionals "
98 "will be merged, and above which conditionals will be split. Set to -1 "
99 "to never merge branches."),
100 cl::Hidden);
101
102static cl::opt<int> BrMergingLikelyBias(
103 "riscv-br-merging-likely-bias", cl::init(Val: 0),
104 cl::desc(
105 "Increases 'riscv-br-merging-base-cost' in cases that it is "
106 "likely that all conditionals will be executed. For example for "
107 "merging the conditionals (a == b && c > d), if its known that "
108 "a == b is likely, then it is likely that if the conditionals are "
109 "split both sides will be executed, so it may be desirable to "
110 "increase the instruction cost threshold. Set to -1 to never merge "
111 "likely branches."),
112 cl::Hidden);
113
114static cl::opt<int> BrMergingUnlikelyBias(
115 "riscv-br-merging-unlikely-bias", cl::init(Val: -1),
116 cl::desc(
117 "Decreases 'riscv-br-merging-base-cost' in cases that it is unlikely "
118 "that all conditionals will be executed. For example for merging "
119 "the conditionals (a == b && c > d), if its known that a == b is "
120 "unlikely, then it is unlikely that if the conditionals are split "
121 "both sides will be executed, so it may be desirable to decrease "
122 "the instruction cost threshold. Set to -1 to never merge unlikely "
123 "branches."),
124 cl::Hidden);
125
126// TODO: Support more ops
127static const unsigned ZvfbfaOps[] = {
128 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FADD,
129 ISD::FSUB, ISD::FMUL, ISD::FMINNUM, ISD::FMAXNUM,
130 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUM, ISD::FMAXIMUM,
131 ISD::FMA, ISD::IS_FPCLASS, ISD::STRICT_FADD, ISD::STRICT_FSUB,
132 ISD::STRICT_FMUL, ISD::STRICT_FMA, ISD::SETCC};
133
134RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
135 const RISCVSubtarget &STI)
136 : TargetLowering(TM, STI), Subtarget(STI) {
137
138 RISCVABI::ABI ABI = Subtarget.getTargetABI();
139 // Note: Hard-float ABIs that don't match the F/D extensions are already
140 // rejected/ by RISCVABI::computeTargetABI() during subtarget construction.
141 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
142
143 switch (ABI) {
144 default:
145 reportFatalUsageError(reason: "Don't know how to lower this ABI");
146 case RISCVABI::ABI_ILP32:
147 case RISCVABI::ABI_ILP32E:
148 case RISCVABI::ABI_LP64E:
149 case RISCVABI::ABI_ILP32F:
150 case RISCVABI::ABI_ILP32D:
151 case RISCVABI::ABI_LP64:
152 case RISCVABI::ABI_LP64F:
153 case RISCVABI::ABI_LP64D:
154 break;
155 }
156
157 MVT XLenVT = Subtarget.getXLenVT();
158
159 // Set up the register classes.
160 addRegisterClass(VT: XLenVT, RC: &RISCV::GPRRegClass);
161
162 if (Subtarget.hasStdExtZfhmin())
163 addRegisterClass(VT: MVT::f16, RC: &RISCV::FPR16RegClass);
164 if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
165 addRegisterClass(VT: MVT::bf16, RC: &RISCV::FPR16RegClass);
166 if (Subtarget.hasStdExtF())
167 addRegisterClass(VT: MVT::f32, RC: &RISCV::FPR32RegClass);
168 if (Subtarget.hasStdExtD())
169 addRegisterClass(VT: MVT::f64, RC: &RISCV::FPR64RegClass);
170 if (Subtarget.hasStdExtZhinxmin())
171 addRegisterClass(VT: MVT::f16, RC: &RISCV::GPRF16RegClass);
172 if (Subtarget.hasStdExtZfinx())
173 addRegisterClass(VT: MVT::f32, RC: &RISCV::GPRF32RegClass);
174 if (Subtarget.hasStdExtZdinx()) {
175 if (Subtarget.is64Bit())
176 addRegisterClass(VT: MVT::f64, RC: &RISCV::GPRRegClass);
177 else
178 addRegisterClass(VT: MVT::f64, RC: &RISCV::GPRPairRegClass);
179 }
180
181 static const MVT::SimpleValueType BoolVecVTs[] = {
182 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
183 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
184 static const MVT::SimpleValueType IntVecVTs[] = {
185 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
186 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
187 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
188 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
189 MVT::nxv4i64, MVT::nxv8i64};
190 static const MVT::SimpleValueType F16VecVTs[] = {
191 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
192 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
193 static const MVT::SimpleValueType BF16VecVTs[] = {
194 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
195 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
196 static const MVT::SimpleValueType F32VecVTs[] = {
197 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
198 static const MVT::SimpleValueType F64VecVTs[] = {
199 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
200 static const MVT::SimpleValueType VecTupleVTs[] = {
201 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
202 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
203 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
204 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
205 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
206 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
207 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
208 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
209 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
210 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
211 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
212
213 if (Subtarget.hasVInstructions()) {
214 auto addRegClassForRVV = [this](MVT VT) {
215 // Disable the smallest fractional LMUL types if ELEN is less than
216 // RVVBitsPerBlock.
217 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
218 if (VT.getVectorMinNumElements() < MinElts)
219 return;
220
221 unsigned Size = VT.getSizeInBits().getKnownMinValue();
222 const TargetRegisterClass *RC;
223 if (Size <= RISCV::RVVBitsPerBlock)
224 RC = &RISCV::VRRegClass;
225 else if (Size == 2 * RISCV::RVVBitsPerBlock)
226 RC = &RISCV::VRM2RegClass;
227 else if (Size == 4 * RISCV::RVVBitsPerBlock)
228 RC = &RISCV::VRM4RegClass;
229 else if (Size == 8 * RISCV::RVVBitsPerBlock)
230 RC = &RISCV::VRM8RegClass;
231 else
232 llvm_unreachable("Unexpected size");
233
234 addRegisterClass(VT, RC);
235 };
236
237 for (MVT VT : BoolVecVTs)
238 addRegClassForRVV(VT);
239 for (MVT VT : IntVecVTs) {
240 if (VT.getVectorElementType() == MVT::i64 &&
241 !Subtarget.hasVInstructionsI64())
242 continue;
243 addRegClassForRVV(VT);
244 }
245
246 if (Subtarget.hasVInstructionsF16Minimal() ||
247 Subtarget.hasVendorXAndesVPackFPH())
248 for (MVT VT : F16VecVTs)
249 addRegClassForRVV(VT);
250
251 if (Subtarget.hasVInstructionsBF16Minimal() ||
252 Subtarget.hasVendorXAndesVBFHCvt())
253 for (MVT VT : BF16VecVTs)
254 addRegClassForRVV(VT);
255
256 if (Subtarget.hasVInstructionsF32())
257 for (MVT VT : F32VecVTs)
258 addRegClassForRVV(VT);
259
260 if (Subtarget.hasVInstructionsF64())
261 for (MVT VT : F64VecVTs)
262 addRegClassForRVV(VT);
263
264 if (Subtarget.useRVVForFixedLengthVectors()) {
265 auto addRegClassForFixedVectors = [this](MVT VT) {
266 MVT ContainerVT = getContainerForFixedLengthVector(VT);
267 unsigned RCID = getRegClassIDForVecVT(VT: ContainerVT);
268 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
269 addRegisterClass(VT, RC: TRI.getRegClass(i: RCID));
270 };
271 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
272 if (useRVVForFixedLengthVectorVT(VT))
273 addRegClassForFixedVectors(VT);
274
275 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
276 if (useRVVForFixedLengthVectorVT(VT))
277 addRegClassForFixedVectors(VT);
278 }
279
280 addRegisterClass(VT: MVT::riscv_nxv1i8x2, RC: &RISCV::VRN2M1RegClass);
281 addRegisterClass(VT: MVT::riscv_nxv1i8x3, RC: &RISCV::VRN3M1RegClass);
282 addRegisterClass(VT: MVT::riscv_nxv1i8x4, RC: &RISCV::VRN4M1RegClass);
283 addRegisterClass(VT: MVT::riscv_nxv1i8x5, RC: &RISCV::VRN5M1RegClass);
284 addRegisterClass(VT: MVT::riscv_nxv1i8x6, RC: &RISCV::VRN6M1RegClass);
285 addRegisterClass(VT: MVT::riscv_nxv1i8x7, RC: &RISCV::VRN7M1RegClass);
286 addRegisterClass(VT: MVT::riscv_nxv1i8x8, RC: &RISCV::VRN8M1RegClass);
287 addRegisterClass(VT: MVT::riscv_nxv2i8x2, RC: &RISCV::VRN2M1RegClass);
288 addRegisterClass(VT: MVT::riscv_nxv2i8x3, RC: &RISCV::VRN3M1RegClass);
289 addRegisterClass(VT: MVT::riscv_nxv2i8x4, RC: &RISCV::VRN4M1RegClass);
290 addRegisterClass(VT: MVT::riscv_nxv2i8x5, RC: &RISCV::VRN5M1RegClass);
291 addRegisterClass(VT: MVT::riscv_nxv2i8x6, RC: &RISCV::VRN6M1RegClass);
292 addRegisterClass(VT: MVT::riscv_nxv2i8x7, RC: &RISCV::VRN7M1RegClass);
293 addRegisterClass(VT: MVT::riscv_nxv2i8x8, RC: &RISCV::VRN8M1RegClass);
294 addRegisterClass(VT: MVT::riscv_nxv4i8x2, RC: &RISCV::VRN2M1RegClass);
295 addRegisterClass(VT: MVT::riscv_nxv4i8x3, RC: &RISCV::VRN3M1RegClass);
296 addRegisterClass(VT: MVT::riscv_nxv4i8x4, RC: &RISCV::VRN4M1RegClass);
297 addRegisterClass(VT: MVT::riscv_nxv4i8x5, RC: &RISCV::VRN5M1RegClass);
298 addRegisterClass(VT: MVT::riscv_nxv4i8x6, RC: &RISCV::VRN6M1RegClass);
299 addRegisterClass(VT: MVT::riscv_nxv4i8x7, RC: &RISCV::VRN7M1RegClass);
300 addRegisterClass(VT: MVT::riscv_nxv4i8x8, RC: &RISCV::VRN8M1RegClass);
301 addRegisterClass(VT: MVT::riscv_nxv8i8x2, RC: &RISCV::VRN2M1RegClass);
302 addRegisterClass(VT: MVT::riscv_nxv8i8x3, RC: &RISCV::VRN3M1RegClass);
303 addRegisterClass(VT: MVT::riscv_nxv8i8x4, RC: &RISCV::VRN4M1RegClass);
304 addRegisterClass(VT: MVT::riscv_nxv8i8x5, RC: &RISCV::VRN5M1RegClass);
305 addRegisterClass(VT: MVT::riscv_nxv8i8x6, RC: &RISCV::VRN6M1RegClass);
306 addRegisterClass(VT: MVT::riscv_nxv8i8x7, RC: &RISCV::VRN7M1RegClass);
307 addRegisterClass(VT: MVT::riscv_nxv8i8x8, RC: &RISCV::VRN8M1RegClass);
308 addRegisterClass(VT: MVT::riscv_nxv16i8x2, RC: &RISCV::VRN2M2RegClass);
309 addRegisterClass(VT: MVT::riscv_nxv16i8x3, RC: &RISCV::VRN3M2RegClass);
310 addRegisterClass(VT: MVT::riscv_nxv16i8x4, RC: &RISCV::VRN4M2RegClass);
311 addRegisterClass(VT: MVT::riscv_nxv32i8x2, RC: &RISCV::VRN2M4RegClass);
312 }
313
314 // fixed vector is stored in GPRs for P extension packed operations
315 if (Subtarget.hasStdExtP()) {
316 if (Subtarget.is64Bit()) {
317 addRegisterClass(VT: MVT::v2i32, RC: &RISCV::GPRRegClass);
318 addRegisterClass(VT: MVT::v4i16, RC: &RISCV::GPRRegClass);
319 addRegisterClass(VT: MVT::v8i8, RC: &RISCV::GPRRegClass);
320 } else {
321 addRegisterClass(VT: MVT::v2i16, RC: &RISCV::GPRRegClass);
322 addRegisterClass(VT: MVT::v4i8, RC: &RISCV::GPRRegClass);
323
324 addRegisterClass(VT: MVT::v2i32, RC: &RISCV::GPRPairRegClass);
325 addRegisterClass(VT: MVT::v4i16, RC: &RISCV::GPRPairRegClass);
326 addRegisterClass(VT: MVT::v8i8, RC: &RISCV::GPRPairRegClass);
327 }
328 }
329
330 // Compute derived properties from the register classes.
331 computeRegisterProperties(TRI: STI.getRegisterInfo());
332
333 setStackPointerRegisterToSaveRestore(RISCV::X2);
334
335 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: XLenVT,
336 MemVT: MVT::i1, Action: Promote);
337 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
338 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
339 MemVT: MVT::i1, Action: Promote);
340
341 // TODO: add all necessary setOperationAction calls.
342 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: XLenVT, Action: Custom);
343
344 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
345 setOperationAction(Op: ISD::BR_CC, VT: XLenVT, Action: Expand);
346 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
347 setOperationAction(Op: ISD::SELECT_CC, VT: XLenVT, Action: Expand);
348
349 setCondCodeAction(CCs: ISD::SETGT, VT: XLenVT, Action: Custom);
350 setCondCodeAction(CCs: ISD::SETGE, VT: XLenVT, Action: Expand);
351 setCondCodeAction(CCs: ISD::SETUGT, VT: XLenVT, Action: Custom);
352 setCondCodeAction(CCs: ISD::SETUGE, VT: XLenVT, Action: Expand);
353 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
354 setCondCodeAction(CCs: ISD::SETULE, VT: XLenVT, Action: Expand);
355 setCondCodeAction(CCs: ISD::SETLE, VT: XLenVT, Action: Expand);
356 }
357
358 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
359
360 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
361 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
362
363 if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
364 !Subtarget.hasVendorXAndesPerf())
365 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
366
367 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
368
369 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
370 !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
371 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
372 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::i8, MVT::i16}, Action: Expand);
373
374 if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
375 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
376 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
377 }
378
379 if (Subtarget.is64Bit()) {
380 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
381
382 setOperationAction(Op: ISD::LOAD, VT: MVT::i32, Action: Custom);
383 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
384 VT: MVT::i32, Action: Custom);
385 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Custom);
386 setOperationAction(Ops: {ISD::SADDO, ISD::SSUBO}, VT: MVT::i32, Action: Custom);
387 } else if (Subtarget.hasStdExtP()) {
388 // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
389 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT: MVT::i64, Action: Custom);
390 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VT: MVT::i64, Action: Custom);
391 }
392 if (!Subtarget.hasStdExtZmmul()) {
393 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU}, VT: XLenVT, Action: Expand);
394 } else if (Subtarget.is64Bit()) {
395 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Custom);
396 setOperationAction(Op: ISD::MUL, VT: MVT::i32, Action: Custom);
397 } else {
398 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
399 }
400
401 if (!Subtarget.hasStdExtM()) {
402 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: XLenVT,
403 Action: Expand);
404 } else if (Subtarget.is64Bit()) {
405 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::UREM},
406 VTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Custom);
407 }
408
409 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT: XLenVT, Action: Expand);
410
411 // On RV32, the P extension has a WMUL(U) instruction we can use for
412 // (S/U)MUL_LOHI.
413 // FIXME: Does P imply Zmmul?
414 if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
415 Subtarget.is64Bit())
416 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: XLenVT, Action: Expand);
417
418 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, VT: XLenVT,
419 Action: Custom);
420
421 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
422 if (Subtarget.is64Bit())
423 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: MVT::i32, Action: Custom);
424 } else if (Subtarget.hasVendorXTHeadBb()) {
425 if (Subtarget.is64Bit())
426 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: MVT::i32, Action: Custom);
427 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: XLenVT, Action: Custom);
428 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
429 setOperationAction(Op: ISD::ROTL, VT: XLenVT, Action: Expand);
430 } else {
431 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: XLenVT, Action: Expand);
432 }
433
434 if (Subtarget.hasStdExtP())
435 setOperationAction(Ops: {ISD::FSHL, ISD::FSHR}, VT: XLenVT, Action: Legal);
436
437 setOperationAction(Op: ISD::BSWAP, VT: XLenVT,
438 Action: Subtarget.hasREV8Like() ? Legal : Expand);
439
440 if (Subtarget.hasREVLike()) {
441 setOperationAction(Op: ISD::BITREVERSE, VT: XLenVT, Action: Legal);
442 } else {
443 // Zbkb can use rev8+brev8 to implement bitreverse.
444 setOperationAction(Op: ISD::BITREVERSE, VT: XLenVT,
445 Action: Subtarget.hasStdExtZbkb() ? Custom : Expand);
446 if (Subtarget.hasStdExtZbkb())
447 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
448 }
449
450 if (Subtarget.hasStdExtZbb() ||
451 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
452 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT: XLenVT,
453 Action: Legal);
454 }
455
456 if (Subtarget.hasCTZLike()) {
457 if (Subtarget.is64Bit())
458 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
459 } else {
460 setOperationAction(Op: ISD::CTTZ, VT: XLenVT, Action: Expand);
461 }
462
463 if (!Subtarget.hasCPOPLike()) {
464 // TODO: These should be set to LibCall, but this currently breaks
465 // the Linux kernel build. See #101786. Lacks i128 tests, too.
466 if (Subtarget.is64Bit())
467 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Expand);
468 else
469 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
470 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
471 }
472
473 if (Subtarget.hasCLZLike()) {
474 // We need the custom lowering to make sure that the resulting sequence
475 // for the 32bit case is efficient on 64bit targets.
476 // Use default promotion for i32 without Zbb.
477 if (Subtarget.is64Bit() &&
478 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
479 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
480 } else {
481 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
482 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: XLenVT, Action: Legal);
483 setOperationAction(Op: ISD::CTLZ, VT: XLenVT, Action: Expand);
484 }
485
486 if (Subtarget.hasStdExtP()) {
487 setOperationAction(Op: ISD::CTLS, VT: XLenVT, Action: Legal);
488 if (Subtarget.is64Bit())
489 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
490 }
491
492 if (Subtarget.hasStdExtP() ||
493 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
494 setOperationAction(Op: ISD::ABS, VT: XLenVT, Action: Legal);
495 if (Subtarget.is64Bit())
496 setOperationAction(Ops: {ISD::ABS, ISD::ABS_MIN_POISON}, VT: MVT::i32, Action: Custom);
497 } else if (Subtarget.hasShortForwardBranchIALU()) {
498 // We can use PseudoCCSUB to implement ABS.
499 setOperationAction(Op: ISD::ABS, VT: XLenVT, Action: Legal);
500 } else if (Subtarget.is64Bit()) {
501 setOperationAction(Ops: {ISD::ABS, ISD::ABS_MIN_POISON}, VT: MVT::i32, Action: Custom);
502 }
503
504 if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
505 setOperationAction(Op: ISD::SELECT, VT: XLenVT, Action: Custom);
506
507 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
508 !Subtarget.is64Bit()) {
509 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
510 VT: MVT::i32, Action: Legal);
511 } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
512 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
513 VT: MVT::i32, Action: Custom);
514 } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
515 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
516 VT: MVT::i32, Action: Custom);
517 }
518
519 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
520 !Subtarget.is64Bit()) {
521 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
522 // pssha.w/psshl.w and extracting.
523 setOperationAction(Op: ISD::SSHLSAT, VT: MVT::i32, Action: Legal);
524 setOperationAction(Op: ISD::USHLSAT, VT: MVT::i32, Action: Legal);
525 }
526
527 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
528 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
529 // paadd.w, paaddu.w and extracting.
530 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VT: MVT::i32, Action: Legal);
531 }
532
533 if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) {
534 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: XLenVT, Action: Legal);
535 if (Subtarget.hasStdExtZbc())
536 setOperationAction(Op: ISD::CLMULR, VT: XLenVT, Action: Legal);
537 } else if (Subtarget.hasStdExtZvbc() && Subtarget.is64Bit()) {
538 // FIXME: Support i32 on RV32 by zexting from XLEN to i64 and extracting
539 // half of the result (low for CLMUL, high for CLMULH).
540 // TODO: Zvbc32e allows us to do a lot more here.
541 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: XLenVT, Action: Custom);
542 }
543
544 static const unsigned FPLegalNodeTypes[] = {
545 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
546 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
547 ISD::LROUND, ISD::LLROUND, ISD::STRICT_LRINT,
548 ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
549 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
550 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
551 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
552
553 static const ISD::CondCode FPCCToExpand[] = {
554 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
555 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
556 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
557
558 static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
559 ISD::FPOW};
560 static const unsigned FPOpToLibCall[] = {ISD::FREM};
561
562 static const unsigned FPRndMode[] = {
563 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
564 ISD::FROUNDEVEN};
565
566 static const unsigned ZfhminZfbfminPromoteOps[] = {
567 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUM,
568 ISD::FMAXIMUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM,
569 ISD::FADD, ISD::FSUB, ISD::FMUL,
570 ISD::FMA, ISD::FDIV, ISD::FSQRT,
571 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
572 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
573 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
574 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC,
575 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN,
576 ISD::FCANONICALIZE};
577
578 if (Subtarget.hasStdExtP()) {
579 static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
580 static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
581 ArrayRef<MVT> VTs;
582 if (Subtarget.is64Bit()) {
583 VTs = P64VecVTs;
584 // There's no instruction for vector shamt in P extension so we unroll to
585 // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
586 // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
587 // widen for those operations that will be unrolled.
588 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA},
589 VTs: {MVT::v2i16, MVT::v4i8}, Action: Custom);
590 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN, VTs: {MVT::v2i16, MVT::v4i8},
591 Action: Custom);
592 // Operand legalization queries the action using the illegal subvector.
593 setOperationAction(Ops: ISD::INSERT_SUBVECTOR, VTs: {MVT::v2i16, MVT::v4i8},
594 Action: Custom);
595 } else {
596 VTs = P32VecVTs;
597 }
598 // By default everything must be expanded.
599 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
600 setOperationAction(Ops: Op, VTs, Action: Expand);
601
602 for (MVT VT : VTs) {
603 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
604 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
605 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
606 MemVT: OtherVT, Action: Expand);
607 }
608 }
609
610 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs, Action: Legal);
611 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VTs, Action: Legal);
612 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VTs, Action: Legal);
613 setOperationAction(Ops: ISD::UADDSAT, VTs, Action: Legal);
614 setOperationAction(Ops: ISD::SADDSAT, VTs, Action: Legal);
615 setOperationAction(Ops: ISD::USUBSAT, VTs, Action: Legal);
616 setOperationAction(Ops: ISD::SSUBSAT, VTs, Action: Legal);
617 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Action: Legal);
618 setOperationAction(Ops: ISD::BITREVERSE, VTs, Action: Legal);
619 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs, Action: Custom);
620 setOperationAction(Ops: ISD::VECTOR_REVERSE, VTs, Action: Legal);
621 for (MVT VT : VTs) {
622 if (VT != MVT::v2i32)
623 setOperationAction(Ops: {ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Action: Legal);
624 if (VT.getVectorElementType() != MVT::i8) {
625 setOperationAction(Op: ISD::SSHLSAT, VT, Action: Custom);
626 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
627 }
628 }
629 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VTs, Action: Legal);
630 setOperationAction(Ops: ISD::SPLAT_VECTOR, VTs, Action: Legal);
631 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs, Action: Legal);
632 setOperationAction(Ops: ISD::SCALAR_TO_VECTOR, VTs, Action: Legal);
633 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Action: Custom);
634 setOperationAction(Ops: ISD::BITCAST, VTs, Action: Custom);
635 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
636 Action: Custom);
637 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
638 Action: Legal);
639 setOperationAction(Ops: ISD::SELECT, VTs, Action: Custom);
640 setOperationAction(Ops: ISD::VSELECT, VTs, Action: Legal);
641 setOperationAction(Ops: ISD::SETCC, VTs, Action: Legal);
642 setCondCodeAction(
643 CCs: {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
644 Action: Expand);
645 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETGT}, VTs, Action: Custom);
646
647 if (!Subtarget.is64Bit())
648 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v4i8}, Action: Custom);
649
650 // P extension vector comparisons produce all 1s for true, all 0s for false
651 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
652
653 if (!Subtarget.is64Bit()) {
654 // By default everything must be expanded.
655 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
656 setOperationAction(Ops: Op, VTs: P64VecVTs, Action: Expand);
657
658 for (MVT VT : P64VecVTs) {
659 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
660 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
661 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
662 MemVT: OtherVT, Action: Expand);
663 }
664 }
665
666 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VTs: P64VecVTs, Action: Legal);
667 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs: P64VecVTs, Action: Custom);
668 setOperationAction(Ops: ISD::BITCAST, VTs: P64VecVTs, Action: Custom);
669 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VTs: P64VecVTs, Action: Legal);
670 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VTs: {MVT::v4i16, MVT::v8i8},
671 Action: Custom);
672 setOperationAction(
673 Ops: {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, VTs: P64VecVTs,
674 Action: Legal);
675 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN, VTs: P64VecVTs, Action: Legal);
676 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
677 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs: P64VecVTs, Action: Legal);
678 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
679 VTs: P64VecVTs, Action: Legal);
680 setOperationAction(Ops: {ISD::ABS, ISD::ABDS, ISD::ABDU},
681 VTs: {MVT::v4i16, MVT::v8i8}, Action: Legal);
682 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VTs: P64VecVTs, Action: Custom);
683 setOperationAction(Ops: ISD::SSHLSAT, VTs: {MVT::v2i32, MVT::v4i16}, Action: Custom);
684 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Legal);
685 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::v4i16, MVT::v8i8}, Action: Legal);
686 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: P64VecVTs, Action: Custom);
687 setOperationAction(Ops: ISD::VECTOR_REVERSE, VTs: P64VecVTs, Action: Custom);
688 setOperationAction(Ops: ISD::SPLAT_VECTOR, VTs: P64VecVTs, Action: Legal);
689 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: P64VecVTs, Action: Legal);
690 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i32, Action: Legal);
691 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
692 VTs: {MVT::v4i16, MVT::v8i8}, Action: Custom);
693 setOperationAction(Ops: ISD::CONCAT_VECTORS, VTs: {MVT::v4i16, MVT::v8i8}, Action: Legal);
694 setOperationAction(Ops: ISD::EXTRACT_SUBVECTOR, VTs: {MVT::v2i16, MVT::v4i8},
695 Action: Legal);
696 setOperationAction(Ops: {ISD::SELECT, ISD::VSELECT}, VTs: {MVT::v4i16, MVT::v8i8},
697 Action: Custom);
698 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU},
699 VTs: {MVT::v4i16, MVT::v8i8}, Action: Custom);
700 setOperationAction(Ops: ISD::MUL, VTs: P32VecVTs, Action: Custom);
701 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT: MVT::v4i8, Action: Custom);
702 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT: MVT::v2i16, Action: Legal);
703 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
704 VTs: {MVT::v4i16, MVT::v2i32}, Action: Legal);
705 setOperationAction(Ops: ISD::TRUNCATE, VTs: {MVT::v4i8, MVT::v2i16}, Action: Legal);
706 setOperationAction(Ops: ISD::SETCC, VTs: P64VecVTs, Action: Legal);
707 setCondCodeAction(
708 CCs: {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
709 VTs: P64VecVTs, Action: Expand);
710 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETGT}, VTs: P64VecVTs, Action: Custom);
711 // Operation legalization queries the action using the result type.
712 setOperationAction(Ops: ISD::INSERT_SUBVECTOR, VTs: {MVT::v4i16, MVT::v8i8},
713 Action: Custom);
714 } else {
715 setOperationAction(Ops: ISD::MUL, VTs: P64VecVTs, Action: Legal);
716 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VTs: {MVT::v2i32, MVT::v4i16},
717 Action: Legal);
718 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT: MVT::v8i8, Action: Custom);
719 setOperationAction(Ops: ISD::ZERO_EXTEND_VECTOR_INREG,
720 VTs: {MVT::v4i16, MVT::v2i32}, Action: Legal);
721 setOperationAction(Ops: ISD::ANY_EXTEND_VECTOR_INREG, VTs: {MVT::v4i16, MVT::v2i32},
722 Action: Custom);
723 setOperationAction(Ops: ISD::TRUNCATE, VTs: {MVT::v4i8, MVT::v2i16}, Action: Custom);
724 }
725 // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
726 setOperationAction(
727 Ops: ISD::SIGN_EXTEND_INREG,
728 VTs: {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}, Action: Legal);
729 }
730
731 if (Subtarget.hasStdExtZfbfmin()) {
732 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
733 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Expand);
734 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::bf16, Action: Expand);
735 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Custom);
736 setOperationAction(Op: ISD::BR_CC, VT: MVT::bf16, Action: Expand);
737 setOperationAction(Ops: ZfhminZfbfminPromoteOps, VT: MVT::bf16, Action: Promote);
738 setOperationAction(Op: ISD::FREM, VT: MVT::bf16, Action: Promote);
739 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Custom);
740 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Custom);
741 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Custom);
742 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: XLenVT, Action: Custom);
743 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: XLenVT, Action: Custom);
744 }
745
746 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
747 if (Subtarget.hasStdExtZfhOrZhinx()) {
748 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f16, Action: Legal);
749 setOperationAction(Ops: FPRndMode, VT: MVT::f16,
750 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
751 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f16, Action: Custom);
752 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16,
753 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
754 if (Subtarget.hasStdExtZfa())
755 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Custom);
756 } else {
757 setOperationAction(Ops: ZfhminZfbfminPromoteOps, VT: MVT::f16, Action: Promote);
758 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16, Action: Promote);
759 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
760 ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
761 ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
762 setOperationAction(Op, VT: MVT::f16, Action: Custom);
763 setOperationAction(Op: ISD::FABS, VT: MVT::f16, Action: Custom);
764 setOperationAction(Op: ISD::FNEG, VT: MVT::f16, Action: Custom);
765 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Custom);
766 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: XLenVT, Action: Custom);
767 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: XLenVT, Action: Custom);
768 }
769
770 if (!Subtarget.hasStdExtD()) {
771 // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
772 // instruction runs into double rounding issues, so this is wrong.
773 // Normally we'd use an f64 fma, but without the D extension the f64 type
774 // is not legal. This should probably be a libcall.
775 AddPromotedToType(Opc: ISD::FMA, OrigVT: MVT::f16, DestVT: MVT::f32);
776 AddPromotedToType(Opc: ISD::STRICT_FMA, OrigVT: MVT::f16, DestVT: MVT::f32);
777 }
778
779 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
780
781 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Legal);
782 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
783 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f16, Action: Expand);
784 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f16, Action: Expand);
785 setOperationAction(Op: ISD::SELECT, VT: MVT::f16, Action: Custom);
786 setOperationAction(Op: ISD::BR_CC, VT: MVT::f16, Action: Expand);
787
788 setOperationAction(
789 Op: ISD::FNEARBYINT, VT: MVT::f16,
790 Action: Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
791 setOperationAction(Ops: {ISD::FREM, ISD::FPOW, ISD::FPOWI,
792 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
793 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
794 ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
795 VT: MVT::f16, Action: Promote);
796
797 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
798 // complete support for all operations in LegalizeDAG.
799 setOperationAction(Ops: {ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
800 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
801 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
802 ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
803 VT: MVT::f16, Action: Promote);
804
805 // We need to custom promote this.
806 if (Subtarget.is64Bit())
807 setOperationAction(Op: ISD::FPOWI, VT: MVT::i32, Action: Custom);
808 }
809
810 if (Subtarget.hasStdExtFOrZfinx()) {
811 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f32, Action: Legal);
812 setOperationAction(Ops: FPRndMode, VT: MVT::f32,
813 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
814 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
815 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
816 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Custom);
817 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
818 setOperationAction(Ops: FPOpToExpand, VT: MVT::f32, Action: Expand);
819 setOperationAction(Ops: FPOpToLibCall, VT: MVT::f32, Action: LibCall);
820 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
821 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
822 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
823 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
824 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
825 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
826 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
827 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
828 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Custom);
829 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Custom);
830 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f32, Action: Custom);
831 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f32, Action: Custom);
832
833 if (Subtarget.hasStdExtZfa()) {
834 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
835 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f32, Action: Legal);
836 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
837 } else {
838 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Custom);
839 }
840 }
841
842 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
843 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
844
845 if (Subtarget.hasStdExtDOrZdinx()) {
846 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f64, Action: Legal);
847
848 if (!Subtarget.is64Bit())
849 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
850
851 if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
852 !Subtarget.is64Bit()) {
853 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Custom);
854 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Custom);
855 }
856
857 if (Subtarget.hasStdExtZfa()) {
858 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
859 setOperationAction(Ops: FPRndMode, VT: MVT::f64, Action: Legal);
860 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f64, Action: Legal);
861 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f64, Action: Legal);
862 } else {
863 if (Subtarget.is64Bit())
864 setOperationAction(Ops: FPRndMode, VT: MVT::f64, Action: Custom);
865
866 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f64, Action: Custom);
867 }
868
869 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
870 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
871 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
872 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
873 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Custom);
874 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
875 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
876 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
877 setOperationAction(Ops: FPOpToExpand, VT: MVT::f64, Action: Expand);
878 setOperationAction(Ops: FPOpToLibCall, VT: MVT::f64, Action: LibCall);
879 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
880 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
881 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
882 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
883 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
884 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
885 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
886 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
887 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Custom);
888 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
889 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f64, Action: Custom);
890 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f64, Action: Expand);
891 }
892
893 if (Subtarget.is64Bit()) {
894 setOperationAction(Ops: {ISD::FP_TO_UINT, ISD::FP_TO_SINT,
895 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
896 VT: MVT::i32, Action: Custom);
897 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
898 } else if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
899 // i64 is not a legal type on RV32, so these are custom expanded in
900 // ReplaceNodeResults: f16 sources are converted to i32 and extended, other
901 // FP sources fall back to the default libcall expansion.
902 setOperationAction(Ops: {ISD::FP_TO_UINT, ISD::FP_TO_SINT,
903 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
904 VT: MVT::i64, Action: Custom);
905 }
906
907 if (Subtarget.hasStdExtFOrZfinx()) {
908 setOperationAction(Ops: {ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, VT: XLenVT,
909 Action: Custom);
910
911 // f16/bf16 require custom handling.
912 setOperationAction(Ops: {ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, VT: XLenVT,
913 Action: Custom);
914 setOperationAction(Ops: {ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, VT: XLenVT,
915 Action: Custom);
916
917 setOperationAction(Op: ISD::GET_ROUNDING, VT: XLenVT, Action: Custom);
918 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
919 setOperationAction(Op: ISD::GET_FPENV, VT: XLenVT, Action: Custom);
920 setOperationAction(Op: ISD::SET_FPENV, VT: XLenVT, Action: Custom);
921 setOperationAction(Op: ISD::RESET_FPENV, VT: MVT::Other, Action: Custom);
922 setOperationAction(Op: ISD::GET_FPMODE, VT: XLenVT, Action: Custom);
923 setOperationAction(Op: ISD::SET_FPMODE, VT: XLenVT, Action: Custom);
924 setOperationAction(Op: ISD::RESET_FPMODE, VT: MVT::Other, Action: Custom);
925 }
926
927 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
928 ISD::JumpTable},
929 VT: XLenVT, Action: Custom);
930
931 setOperationAction(Op: ISD::GlobalTLSAddress, VT: XLenVT, Action: Custom);
932
933 if (Subtarget.is64Bit())
934 setOperationAction(Op: ISD::Constant, VT: MVT::i64, Action: Custom);
935
936 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
937 // Unfortunately this can't be determined just from the ISA naming string.
938 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64,
939 Action: Subtarget.is64Bit() ? Legal : Custom);
940 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64,
941 Action: Subtarget.is64Bit() ? Legal : Custom);
942
943 if (Subtarget.is64Bit()) {
944 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
945 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
946 }
947
948 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Legal);
949 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
950 if (Subtarget.is64Bit())
951 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
952
953 if (Subtarget.hasVendorXMIPSCBOP())
954 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
955 else
956 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
957
958 if (Subtarget.hasStdExtZalrsc()) {
959 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
960 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
961 setMinCmpXchgSizeInBits(8);
962 else
963 setMinCmpXchgSizeInBits(32);
964 } else if (Subtarget.hasForcedAtomics()) {
965 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
966 } else {
967 setMaxAtomicSizeInBitsSupported(0);
968 }
969
970 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
971
972 setBooleanContents(ZeroOrOneBooleanContent);
973
974 if (getTargetMachine().getTargetTriple().isOSLinux()) {
975 // Custom lowering of llvm.clear_cache.
976 setOperationAction(Op: ISD::CLEAR_CACHE, VT: MVT::Other, Action: Custom);
977 }
978
979 if (Subtarget.hasVInstructions()) {
980 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
981
982 setOperationAction(Op: ISD::VSCALE, VT: XLenVT, Action: Custom);
983
984 // RVV intrinsics may have illegal operands.
985 // We also need to custom legalize vmv.x.s.
986 setOperationAction(Ops: {ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
987 ISD::INTRINSIC_VOID},
988 VTs: {MVT::i8, MVT::i16}, Action: Custom);
989 if (Subtarget.is64Bit())
990 setOperationAction(Ops: {ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
991 VT: MVT::i32, Action: Custom);
992 else
993 setOperationAction(Ops: {ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
994 VT: MVT::i64, Action: Custom);
995
996 setOperationAction(Ops: {ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
997 VT: MVT::Other, Action: Custom);
998
999 static const unsigned IntegerVPOps[] = {
1000 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
1001 ISD::VP_UREM, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1002 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
1003 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
1004 ISD::VP_MERGE,
1005 ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
1006 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON};
1007
1008 static const unsigned FloatingPointVPOps[] = {
1009 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
1010 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
1011 ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
1012
1013 static const unsigned IntegerVecReduceOps[] = {
1014 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
1015 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
1016 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
1017
1018 static const unsigned FloatingPointVecReduceOps[] = {
1019 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
1020 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
1021
1022 static const unsigned FloatingPointLibCallOps[] = {
1023 ISD::FREM, ISD::FPOW, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
1024 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
1025
1026 if (!Subtarget.is64Bit()) {
1027 // We must custom-lower certain vXi64 operations on RV32 due to the vector
1028 // element type being illegal.
1029 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1030 VT: MVT::i64, Action: Custom);
1031
1032 setOperationAction(Ops: IntegerVecReduceOps, VT: MVT::i64, Action: Custom);
1033
1034 setOperationAction(Ops: {ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1035 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
1036 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
1037 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
1038 VT: MVT::i64, Action: Custom);
1039 }
1040
1041 for (MVT VT : BoolVecVTs) {
1042 if (!isTypeLegal(VT))
1043 continue;
1044
1045 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Custom);
1046
1047 // Mask VTs are custom-expanded into a series of standard nodes
1048 setOperationAction(Ops: {ISD::TRUNCATE, ISD::CONCAT_VECTORS,
1049 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1050 ISD::SCALAR_TO_VECTOR},
1051 VT, Action: Custom);
1052
1053 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1054 Action: Custom);
1055
1056 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1057 setOperationAction(Ops: {ISD::SELECT_CC, ISD::VSELECT}, VT,
1058 Action: Expand);
1059 setOperationAction(Op: ISD::VP_MERGE, VT, Action: Custom);
1060
1061 setOperationAction(Ops: {ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
1062 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
1063 VT, Action: Custom);
1064
1065 setOperationAction(
1066 Ops: {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1067 Action: Custom);
1068
1069 setOperationAction(
1070 Ops: {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1071 Action: Custom);
1072
1073 // RVV has native int->float & float->int conversions where the
1074 // element type sizes are within one power-of-two of each other. Any
1075 // wider distances between type sizes have to be lowered as sequences
1076 // which progressively narrow the gap in stages.
1077 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1078 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1079 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1080 ISD::STRICT_FP_TO_UINT},
1081 VT, Action: Custom);
1082 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1083 Action: Custom);
1084
1085 // Expand all extending loads to types larger than this, and truncating
1086 // stores from types larger than this.
1087 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1088 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1089 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1090 MemVT: OtherVT, Action: Expand);
1091 }
1092
1093 setVectorInterleaveAction(
1094 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1095 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1096
1097 setOperationAction(Op: ISD::VECTOR_REVERSE, VT, Action: Custom);
1098
1099 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1100 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1101
1102 setOperationPromotedToType(
1103 Ops: {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, OrigVT: VT,
1104 DestVT: MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount()));
1105 }
1106
1107 for (MVT VT : IntVecVTs) {
1108 if (!isTypeLegal(VT))
1109 continue;
1110
1111 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1112 setOperationAction(Op: ISD::SPLAT_VECTOR_PARTS, VT, Action: Custom);
1113
1114 // Vectors implement MULHS/MULHU.
1115 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
1116
1117 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1118 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
1119 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT, Action: Expand);
1120
1121 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
1122 Action: Legal);
1123
1124 if (Subtarget.hasStdExtZvabd()) {
1125 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
1126 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Legal);
1127 } else {
1128 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1129 }
1130
1131 // Custom-lower extensions and truncations from/to mask types.
1132 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
1133 VT, Action: Custom);
1134
1135 // RVV has native int->float & float->int conversions where the
1136 // element type sizes are within one power-of-two of each other. Any
1137 // wider distances between type sizes have to be lowered as sequences
1138 // which progressively narrow the gap in stages.
1139 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1140 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1141 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1142 ISD::STRICT_FP_TO_UINT},
1143 VT, Action: Custom);
1144 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1145 Action: Custom);
1146 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1147 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1148 ISD::SSUBSAT, ISD::USUBSAT},
1149 VT, Action: Legal);
1150
1151 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
1152 // nodes which truncate by one power of two at a time.
1153 setOperationAction(
1154 Ops: {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1155 Action: Custom);
1156
1157 // Custom-lower insert/extract operations to simplify patterns.
1158 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1159 Action: Custom);
1160
1161 // Custom-lower reduction operations to set up the corresponding custom
1162 // nodes' operands.
1163 setOperationAction(Ops: IntegerVecReduceOps, VT, Action: Custom);
1164
1165 setOperationAction(Ops: IntegerVPOps, VT, Action: Custom);
1166
1167 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1168
1169 setOperationAction(Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1170 VT, Action: Custom);
1171
1172 setOperationAction(
1173 Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1174 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1175 VT, Action: Custom);
1176 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1177
1178 setOperationAction(Ops: {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1179 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1180 VT, Action: Custom);
1181
1182 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1183 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1184
1185 setOperationAction(Ops: {ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Action: Custom);
1186
1187 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1188 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1189 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1190 MemVT: OtherVT, Action: Expand);
1191 }
1192
1193 setVectorInterleaveAction(
1194 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1195 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1196
1197 setOperationAction(Ops: {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
1198 VT, Action: Custom);
1199
1200 if (Subtarget.hasStdExtZvkb()) {
1201 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
1202 } else {
1203 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
1204 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT, Action: Expand);
1205 }
1206
1207 if (Subtarget.hasStdExtZvbb()) {
1208 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Legal);
1209 } else {
1210 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Expand);
1211 setOperationAction(Ops: {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Action: Expand);
1212
1213 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1214 // range of f32.
1215 EVT FloatVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1216 if (isTypeLegal(VT: FloatVT)) {
1217 setOperationAction(
1218 Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1219 Action: Custom);
1220 }
1221 }
1222
1223 if (VT.getVectorElementType() == MVT::i64) {
1224 if (Subtarget.hasStdExtZvbc())
1225 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Legal);
1226 } else {
1227 if (Subtarget.hasStdExtZvbc32e()) {
1228 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Legal);
1229 } else if (Subtarget.hasStdExtZvbc()) {
1230 // Promote to i64 if the lmul is small enough.
1231 // FIXME: Split if necessary to widen.
1232 // FIXME: Promote clmulh directly without legalizing to clmul first.
1233 MVT I64VecVT = MVT::getVectorVT(VT: MVT::i64, EC: VT.getVectorElementCount());
1234 if (isTypeLegal(VT: I64VecVT))
1235 setOperationAction(Op: ISD::CLMUL, VT, Action: Custom);
1236 }
1237 }
1238
1239 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1240 setOperationAction(Ops: {ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
1241 ISD::MASKED_SREM},
1242 VT, Action: Legal);
1243 }
1244
1245 for (MVT VT : VecTupleVTs) {
1246 if (!isTypeLegal(VT))
1247 continue;
1248
1249 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1250 }
1251
1252 // Expand various CCs to best match the RVV ISA, which natively supports UNE
1253 // but no other unordered comparisons, and supports all ordered comparisons
1254 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
1255 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
1256 // and we pattern-match those back to the "original", swapping operands once
1257 // more. This way we catch both operations and both "vf" and "fv" forms with
1258 // fewer patterns.
1259 static const ISD::CondCode VFPCCToExpand[] = {
1260 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
1261 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
1262 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
1263 };
1264
1265 // TODO: support more ops.
1266 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
1267 ISD::FMINNUM,
1268 ISD::FMAXNUM,
1269 ISD::FMINIMUMNUM,
1270 ISD::FMAXIMUMNUM,
1271 ISD::FADD,
1272 ISD::FSUB,
1273 ISD::FMUL,
1274 ISD::FMA,
1275 ISD::FDIV,
1276 ISD::FSQRT,
1277 ISD::FCEIL,
1278 ISD::FTRUNC,
1279 ISD::FFLOOR,
1280 ISD::FROUND,
1281 ISD::FROUNDEVEN,
1282 ISD::FRINT,
1283 ISD::FNEARBYINT,
1284 ISD::IS_FPCLASS,
1285 ISD::SETCC,
1286 ISD::FMAXIMUM,
1287 ISD::FMINIMUM,
1288 ISD::STRICT_FADD,
1289 ISD::STRICT_FSUB,
1290 ISD::STRICT_FMUL,
1291 ISD::STRICT_FDIV,
1292 ISD::STRICT_FSQRT,
1293 ISD::STRICT_FMA,
1294 ISD::VECREDUCE_FADD,
1295 ISD::VECREDUCE_FMIN,
1296 ISD::VECREDUCE_FMAX,
1297 ISD::VECREDUCE_FMINIMUM,
1298 ISD::VECREDUCE_FMAXIMUM,
1299 ISD::FCANONICALIZE};
1300
1301 // TODO: Make more of these ops legal.
1302 static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
1303 ISD::FSQRT,
1304 ISD::FCEIL,
1305 ISD::FTRUNC,
1306 ISD::FFLOOR,
1307 ISD::FROUND,
1308 ISD::FROUNDEVEN,
1309 ISD::FRINT,
1310 ISD::FNEARBYINT,
1311 ISD::STRICT_FDIV,
1312 ISD::STRICT_FSQRT,
1313 ISD::VECREDUCE_FADD,
1314 ISD::VECREDUCE_FMIN,
1315 ISD::VECREDUCE_FMAX,
1316 ISD::VECREDUCE_FMINIMUM,
1317 ISD::VECREDUCE_FMAXIMUM};
1318
1319 // TODO: support more vp ops.
1320 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
1321 ISD::VP_REDUCE_FMIN,
1322 ISD::VP_REDUCE_FMAX,
1323 ISD::VP_REDUCE_FMINIMUM,
1324 ISD::VP_REDUCE_FMAXIMUM};
1325
1326 // Sets common operation actions on RVV floating-point vector types.
1327 const auto SetCommonVFPActions = [&](MVT VT) {
1328 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1329 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
1330 // sizes are within one power-of-two of each other. Therefore conversions
1331 // between vXf16 and vXf64 must be lowered as sequences which convert via
1332 // vXf32.
1333 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1334 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1335 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1336 // Custom-lower insert/extract operations to simplify patterns.
1337 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1338 Action: Custom);
1339 // Expand various condition codes (explained above).
1340 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1341
1342 setOperationAction(
1343 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1344 Action: Legal);
1345 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Action: Custom);
1346
1347 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1348 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1349 ISD::IS_FPCLASS},
1350 VT, Action: Custom);
1351
1352 setOperationAction(Ops: FloatingPointVecReduceOps, VT, Action: Custom);
1353
1354 // Expand FP operations that need libcalls.
1355 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1356
1357 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Expand);
1358
1359 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Legal);
1360
1361 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1362
1363 setOperationAction(Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1364 VT, Action: Custom);
1365
1366 setOperationAction(
1367 Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1368 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1369 VT, Action: Custom);
1370 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1371
1372 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1373 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1374
1375 setOperationAction(Ops: {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1376 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1377 VT, Action: Custom);
1378
1379 setVectorInterleaveAction(
1380 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1381 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1382
1383 setOperationAction(Ops: {ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1384 ISD::VECTOR_SPLICE_RIGHT},
1385 VT, Action: Custom);
1386 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1387 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1388
1389 setOperationAction(Ops: FloatingPointVPOps, VT, Action: Custom);
1390
1391 setOperationAction(Ops: {ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1392 Action: Custom);
1393 setOperationAction(Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1394 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1395 VT, Action: Legal);
1396 setOperationAction(Ops: {ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1397 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1398 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1399 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1400 VT, Action: Custom);
1401
1402 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1403 };
1404
1405 // Sets common extload/truncstore actions on RVV floating-point vector
1406 // types.
1407 const auto SetCommonVFPExtLoadTruncStoreActions =
1408 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1409 for (auto SmallVT : SmallerVTs) {
1410 setTruncStoreAction(ValVT: VT, MemVT: SmallVT, Action: Expand);
1411 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: SmallVT, Action: Expand);
1412 }
1413 };
1414
1415 // Sets common actions for f16 and bf16 for when there's only
1416 // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
1417 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1418 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1419 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1420 Action: Custom);
1421 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1422 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1423 setOperationAction(Ops: {ISD::VP_MERGE, ISD::SELECT}, VT,
1424 Action: Custom);
1425 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1426 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
1427 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1428 ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1429 ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
1430 VT, Action: Custom);
1431 setVectorInterleaveAction(
1432 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1433 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1434 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1435 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1436 MVT EltVT = VT.getVectorElementType();
1437 if (isTypeLegal(VT: EltVT))
1438 setOperationAction(Ops: {ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
1439 VT, Action: Custom);
1440 else
1441 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: EltVT, Action: Custom);
1442 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1443 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1444 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1445 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1446 ISD::VP_SCATTER},
1447 VT, Action: Custom);
1448 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1449
1450 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
1451 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
1452 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
1453
1454 // Expand FP operations that need libcalls.
1455 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1456
1457 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Expand);
1458
1459 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1460 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1461 setOperationAction(Ops: ZvfhminZvfbfminPromoteOps, VT, Action: Custom);
1462 setOperationAction(Ops: ZvfhminZvfbfminPromoteVPOps, VT, Action: Custom);
1463 } else {
1464 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1465 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1466 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1467 }
1468 };
1469
1470 // Sets common actions for zvfbfa, some of instructions are supported
1471 // natively so that we don't need to promote them.
1472 const auto SetZvfbfaActions = [&](MVT VT) {
1473 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1474 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1475 Action: Custom);
1476 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1477 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1478 setOperationAction(Ops: {ISD::VP_MERGE, ISD::SELECT}, VT,
1479 Action: Custom);
1480 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1481 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1482 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1483 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1484 ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
1485 ISD::VECTOR_COMPRESS},
1486 VT, Action: Custom);
1487 setVectorInterleaveAction(
1488 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1489 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1490 setOperationAction(
1491 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1492 Action: Legal);
1493 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Action: Custom);
1494 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
1495 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1496 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1497
1498 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Legal);
1499 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1500 setOperationAction(Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1501 ISD::STRICT_FMA},
1502 VT, Action: Legal);
1503 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1504
1505 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1506 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1507 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1508 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1509 ISD::VP_SCATTER},
1510 VT, Action: Custom);
1511 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1512
1513 // Expand FP operations that need libcalls.
1514 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1515
1516 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Expand);
1517
1518 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1519 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1520 setOperationAction(Ops: ZvfbfaPromoteOps, VT, Action: Custom);
1521 setOperationAction(Ops: ZvfhminZvfbfminPromoteVPOps, VT, Action: Custom);
1522 } else {
1523 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1524 setOperationPromotedToType(Ops: ZvfbfaPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1525 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1526 }
1527 };
1528
1529 if (Subtarget.hasVInstructionsF16()) {
1530 for (MVT VT : F16VecVTs) {
1531 if (!isTypeLegal(VT))
1532 continue;
1533 SetCommonVFPActions(VT);
1534 }
1535 } else if (Subtarget.hasVInstructionsF16Minimal()) {
1536 for (MVT VT : F16VecVTs) {
1537 if (!isTypeLegal(VT))
1538 continue;
1539 SetCommonPromoteToF32Actions(VT);
1540 }
1541 }
1542
1543 if (Subtarget.hasVInstructionsBF16()) {
1544 for (MVT VT : BF16VecVTs) {
1545 if (!isTypeLegal(VT))
1546 continue;
1547 SetZvfbfaActions(VT);
1548 }
1549 } else if (Subtarget.hasVInstructionsBF16Minimal()) {
1550 for (MVT VT : BF16VecVTs) {
1551 if (!isTypeLegal(VT))
1552 continue;
1553 SetCommonPromoteToF32Actions(VT);
1554 }
1555 }
1556
1557 if (Subtarget.hasStdExtZvfofp8min()) {
1558 for (MVT VT : BF16VecVTs) {
1559 if (!isTypeLegal(VT))
1560 continue;
1561 setOperationAction(Op: ISD::CONVERT_FROM_ARBITRARY_FP, VT, Action: Custom);
1562 }
1563 }
1564
1565 if (Subtarget.hasVInstructionsF32()) {
1566 for (MVT VT : F32VecVTs) {
1567 if (!isTypeLegal(VT))
1568 continue;
1569 SetCommonVFPActions(VT);
1570 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1571 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1572 }
1573 }
1574
1575 if (Subtarget.hasVInstructionsF64()) {
1576 for (MVT VT : F64VecVTs) {
1577 if (!isTypeLegal(VT))
1578 continue;
1579 SetCommonVFPActions(VT);
1580 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1581 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1582 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1583 }
1584 }
1585
1586 if (Subtarget.useRVVForFixedLengthVectors()) {
1587 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1588 if (!useRVVForFixedLengthVectorVT(VT))
1589 continue;
1590
1591 // By default everything must be expanded.
1592 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1593 setOperationAction(Op, VT, Action: Expand);
1594 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
1595 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1596 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1597 MemVT: OtherVT, Action: Expand);
1598 }
1599
1600 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1601 // expansion to a build_vector of 0s.
1602 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Custom);
1603
1604 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1605 setOperationAction(Ops: {ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1606 Action: Custom);
1607
1608 setOperationAction(
1609 Ops: {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1610 Action: Custom);
1611
1612 setVectorInterleaveAction(
1613 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1614 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1615
1616 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1617 VT, Action: Custom);
1618
1619 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
1620
1621 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1622
1623 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1624
1625 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1626
1627 setOperationAction(
1628 Ops: {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1629 Action: Custom);
1630
1631 setOperationAction(Op: ISD::BITCAST, VT, Action: Custom);
1632
1633 setOperationAction(
1634 Ops: {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1635 Action: Custom);
1636
1637 setOperationAction(
1638 Ops: {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1639 Action: Custom);
1640
1641 setOperationAction(
1642 Ops: {
1643 ISD::SINT_TO_FP,
1644 ISD::UINT_TO_FP,
1645 ISD::FP_TO_SINT,
1646 ISD::FP_TO_UINT,
1647 ISD::STRICT_SINT_TO_FP,
1648 ISD::STRICT_UINT_TO_FP,
1649 ISD::STRICT_FP_TO_SINT,
1650 ISD::STRICT_FP_TO_UINT,
1651 },
1652 VT, Action: Custom);
1653 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1654 Action: Custom);
1655
1656 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
1657
1658 // Operations below are different for between masks and other vectors.
1659 if (VT.getVectorElementType() == MVT::i1) {
1660 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Custom);
1661
1662 setOperationAction(Op: ISD::VP_MERGE, VT, Action: Custom);
1663
1664 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1665 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1666
1667 setOperationAction(Ops: {ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
1668 Action: Custom);
1669 continue;
1670 }
1671
1672 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1673 // it before type legalization for i64 vectors on RV32. It will then be
1674 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1675 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1676 // improvements first.
1677 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1678 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1679 setOperationAction(Op: ISD::SPLAT_VECTOR_PARTS, VT, Action: Custom);
1680
1681 // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1682 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::i64, Action: Custom);
1683 }
1684
1685 setOperationAction(
1686 Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Action: Custom);
1687
1688 setOperationAction(Ops: {ISD::VP_LOAD, ISD::VP_STORE,
1689 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1690 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1691 ISD::VP_SCATTER},
1692 VT, Action: Custom);
1693 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1694
1695 setOperationAction(Ops: {ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1696 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1697 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1698 VT, Action: Custom);
1699
1700 setOperationAction(
1701 Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Action: Custom);
1702
1703 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1704
1705 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1706 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1707 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Custom);
1708
1709 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1710 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1711 ISD::SSUBSAT, ISD::USUBSAT},
1712 VT, Action: Custom);
1713
1714 setOperationAction(Op: ISD::VSELECT, VT, Action: Custom);
1715
1716 setOperationAction(
1717 Ops: {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Action: Custom);
1718
1719 // Custom-lower reduction operations to set up the corresponding custom
1720 // nodes' operands.
1721 setOperationAction(Ops: {ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1722 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1723 ISD::VECREDUCE_UMIN},
1724 VT, Action: Custom);
1725
1726 setOperationAction(Ops: IntegerVPOps, VT, Action: Custom);
1727
1728 if (Subtarget.hasStdExtZvkb())
1729 setOperationAction(Ops: {ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Action: Custom);
1730
1731 if (Subtarget.hasStdExtZvbb()) {
1732 setOperationAction(Ops: {ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
1733 ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
1734 VT, Action: Custom);
1735 } else {
1736 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1737 // range of f32.
1738 EVT FloatVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1739 if (isTypeLegal(VT: FloatVT))
1740 setOperationAction(
1741 Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1742 Action: Custom);
1743 }
1744
1745 if (VT.getVectorElementType() == MVT::i64) {
1746 if (Subtarget.hasStdExtZvbc())
1747 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Custom);
1748 } else {
1749 if (Subtarget.hasStdExtZvbc32e()) {
1750 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Custom);
1751 } else if (Subtarget.hasStdExtZvbc()) {
1752 // Promote to i64 as is done for scalable vectors.
1753 MVT I64VecVT =
1754 MVT::getVectorVT(VT: MVT::i64, EC: VT.getVectorElementCount());
1755 if (I64VecVT.isValid() && useRVVForFixedLengthVectorVT(VT: I64VecVT))
1756 setOperationAction(Op: ISD::CLMUL, VT, Action: Custom);
1757 }
1758 }
1759
1760 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1761 setOperationAction(Ops: {ISD::MASKED_UDIV, ISD::MASKED_SDIV,
1762 ISD::MASKED_UREM, ISD::MASKED_SREM},
1763 VT, Action: Custom);
1764 }
1765
1766 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1767 // There are no extending loads or truncating stores.
1768 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1769 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
1770 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
1771 }
1772
1773 if (!useRVVForFixedLengthVectorVT(VT))
1774 continue;
1775
1776 // By default everything must be expanded.
1777 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1778 setOperationAction(Op, VT, Action: Expand);
1779
1780 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1781 // expansion to a build_vector of 0s.
1782 setOperationAction(Ops: {ISD::UNDEF, ISD::POISON}, VT, Action: Custom);
1783
1784 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1785 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1786 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1787 ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
1788 VT, Action: Custom);
1789 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1790 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1791
1792 setVectorInterleaveAction(
1793 Opcodes: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1794 Factors: {2, 3, 4, 5, 6, 7, 8}, VT, Action: Custom);
1795
1796 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1797 ISD::MGATHER, ISD::MSCATTER},
1798 VT, Action: Custom);
1799 setOperationAction(Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
1800 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1801 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1802 VT, Action: Custom);
1803 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1804
1805 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1806 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1807 Action: Custom);
1808
1809 setOperationAction(Op: ISD::BITCAST, VT, Action: Custom);
1810
1811 if (VT.getVectorElementType() == MVT::f16 &&
1812 !Subtarget.hasVInstructionsF16()) {
1813 setOperationAction(
1814 Ops: {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1815 Action: Custom);
1816 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1817 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1818 if (Subtarget.hasStdExtZfhmin()) {
1819 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
1820 } else {
1821 // We need to custom legalize f16 build vectors if Zfhmin isn't
1822 // available.
1823 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
1824 }
1825 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
1826 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
1827 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
1828 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1829 // Don't promote f16 vector operations to f32 if f32 vector type is
1830 // not legal.
1831 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1832 // TODO: Support more operations.
1833 if (!isTypeLegal(VT: F32VecVT)) {
1834 setOperationAction(Ops: {ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1835 ISD::VECREDUCE_FMAXIMUM,
1836 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1837 VT, Action: Custom);
1838 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1839 continue;
1840 }
1841 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1842 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1843 continue;
1844 }
1845
1846 if (VT.getVectorElementType() == MVT::bf16) {
1847 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1848 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1849 if (Subtarget.hasStdExtZvfofp8min())
1850 setOperationAction(Op: ISD::CONVERT_FROM_ARBITRARY_FP, VT, Action: Custom);
1851 if (Subtarget.hasStdExtZfbfmin()) {
1852 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
1853 } else {
1854 // We need to custom legalize bf16 build vectors if Zfbfmin isn't
1855 // available.
1856 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
1857 }
1858 if (Subtarget.hasVInstructionsBF16()) {
1859 setOperationAction(Ops: ZvfbfaOps, VT, Action: Custom);
1860 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1861 }
1862 setOperationAction(
1863 Ops: {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1864 Action: Custom);
1865 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1866 // Don't promote bf16 vector operations to f32 if f32 vector type is
1867 // not legal.
1868 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1869 // TODO: Support more operations.
1870 if (!isTypeLegal(VT: F32VecVT)) {
1871 setOperationAction(Ops: {ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1872 ISD::VECREDUCE_FMAXIMUM,
1873 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1874 VT, Action: Custom);
1875 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1876 continue;
1877 }
1878
1879 if (Subtarget.hasVInstructionsBF16())
1880 setOperationPromotedToType(Ops: ZvfbfaPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1881 else
1882 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1883 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1884 continue;
1885 }
1886
1887 setOperationAction(Ops: {ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1888 Action: Custom);
1889
1890 setOperationAction(Ops: {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1891 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1892 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1893 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
1894 ISD::FMAXIMUM, ISD::FMINIMUM},
1895 VT, Action: Custom);
1896
1897 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1898 ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
1899 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
1900 ISD::FNEARBYINT, ISD::FCANONICALIZE},
1901 VT, Action: Custom);
1902
1903 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1904
1905 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1906 setOperationAction(Ops: {ISD::VSELECT, ISD::SELECT}, VT, Action: Custom);
1907
1908 setOperationAction(Ops: FloatingPointVecReduceOps, VT, Action: Custom);
1909
1910 setOperationAction(Ops: FloatingPointVPOps, VT, Action: Custom);
1911
1912 setOperationAction(
1913 Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1914 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1915 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1916 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1917 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1918 VT, Action: Custom);
1919 }
1920
1921 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1922 setOperationAction(Ops: ISD::BITCAST, VTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Custom);
1923 if (Subtarget.is64Bit())
1924 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
1925 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1926 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
1927 if (Subtarget.hasStdExtZfbfmin())
1928 setOperationAction(Op: ISD::BITCAST, VT: MVT::bf16, Action: Custom);
1929 if (Subtarget.hasStdExtFOrZfinx())
1930 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
1931 if (Subtarget.hasStdExtDOrZdinx())
1932 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Custom);
1933 }
1934 }
1935
1936 if (Subtarget.hasStdExtZaamo())
1937 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: XLenVT, Action: Expand);
1938
1939 if (Subtarget.hasForcedAtomics()) {
1940 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1941 setOperationAction(
1942 Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1943 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1944 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1945 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1946 VT: XLenVT, Action: LibCall);
1947 }
1948
1949 if (Subtarget.hasVendorXTHeadMemIdx()) {
1950 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1951 setIndexedLoadAction(IdxModes: im, VT: MVT::i8, Action: Legal);
1952 setIndexedStoreAction(IdxModes: im, VT: MVT::i8, Action: Legal);
1953 setIndexedLoadAction(IdxModes: im, VT: MVT::i16, Action: Legal);
1954 setIndexedStoreAction(IdxModes: im, VT: MVT::i16, Action: Legal);
1955 setIndexedLoadAction(IdxModes: im, VT: MVT::i32, Action: Legal);
1956 setIndexedStoreAction(IdxModes: im, VT: MVT::i32, Action: Legal);
1957
1958 if (Subtarget.is64Bit()) {
1959 setIndexedLoadAction(IdxModes: im, VT: MVT::i64, Action: Legal);
1960 setIndexedStoreAction(IdxModes: im, VT: MVT::i64, Action: Legal);
1961 }
1962 }
1963 }
1964
1965 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1966 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i8, Action: Legal);
1967 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i16, Action: Legal);
1968 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
1969
1970 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i8, Action: Legal);
1971 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i16, Action: Legal);
1972 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
1973 }
1974
1975 // zve32x is broken for partial_reduce_umla, but let's not make it worse.
1976 if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
1977 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
1978 ISD::PARTIAL_REDUCE_UMLA,
1979 ISD::PARTIAL_REDUCE_SUMLA};
1980 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv1i32, InputVT: MVT::nxv4i8, Action: Custom);
1981 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv2i32, InputVT: MVT::nxv8i8, Action: Custom);
1982 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv4i32, InputVT: MVT::nxv16i8, Action: Custom);
1983 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv8i32, InputVT: MVT::nxv32i8, Action: Custom);
1984 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv16i32, InputVT: MVT::nxv64i8, Action: Custom);
1985
1986 // An i64 accumulator is handled by performing an i32 vdot4a* and widening
1987 // the result to i64 (see lowerPARTIAL_REDUCE_MLA).
1988 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv1i64, InputVT: MVT::nxv8i8, Action: Custom);
1989 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv2i64, InputVT: MVT::nxv16i8, Action: Custom);
1990 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv4i64, InputVT: MVT::nxv32i8, Action: Custom);
1991 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv8i64, InputVT: MVT::nxv64i8, Action: Custom);
1992
1993 if (Subtarget.useRVVForFixedLengthVectors()) {
1994 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1995 if ((VT.getVectorElementType() != MVT::i32 &&
1996 VT.getVectorElementType() != MVT::i64) ||
1997 !useRVVForFixedLengthVectorVT(VT))
1998 continue;
1999 ElementCount EC = VT.getVectorElementCount();
2000 unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4;
2001 MVT ArgVT = MVT::getVectorVT(VT: MVT::i8, EC: EC * Scale);
2002 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: VT, InputVT: ArgVT, Action: Custom);
2003 }
2004 }
2005 }
2006
2007 // Customize load and store operation for bf16 if zfh isn't enabled.
2008 if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
2009 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Custom);
2010 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Custom);
2011 }
2012
2013 // Function alignments.
2014 const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
2015 setMinFunctionAlignment(FunctionAlignment);
2016 // Set preferred alignments.
2017 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
2018 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
2019
2020 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
2021 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
2022 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
2023 ISD::SRA});
2024 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
2025
2026 if (Subtarget.hasStdExtFOrZfinx())
2027 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
2028
2029 // Allow scalar min/max to be combined with vector reductions.
2030 if (Subtarget.hasVInstructions())
2031 setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
2032 if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
2033 setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
2034
2035 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
2036 Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
2037 setTargetDAGCombine(ISD::TRUNCATE);
2038
2039 if (Subtarget.hasStdExtZbkb())
2040 setTargetDAGCombine(ISD::BITREVERSE);
2041
2042 if (Subtarget.hasStdExtFOrZfinx())
2043 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2044 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
2045 if (Subtarget.hasVInstructions())
2046 setTargetDAGCombine({ISD::FCOPYSIGN,
2047 ISD::VECTOR_MATCH,
2048 ISD::MGATHER,
2049 ISD::MSCATTER,
2050 ISD::MLOAD,
2051 ISD::VP_GATHER,
2052 ISD::VP_SCATTER,
2053 ISD::SRL,
2054 ISD::SHL,
2055 ISD::STORE,
2056 ISD::SPLAT_VECTOR,
2057 ISD::BUILD_VECTOR,
2058 ISD::CONCAT_VECTORS,
2059 ISD::VP_STORE,
2060 ISD::EXPERIMENTAL_VP_REVERSE,
2061 ISD::SDIV,
2062 ISD::UDIV,
2063 ISD::SREM,
2064 ISD::UREM,
2065 ISD::INSERT_VECTOR_ELT,
2066 ISD::ABS,
2067 ISD::ABS_MIN_POISON,
2068 ISD::CTPOP,
2069 ISD::VECTOR_SHUFFLE,
2070 ISD::FMA,
2071 ISD::VSELECT,
2072 ISD::VECREDUCE_ADD,
2073 ISD::VECTOR_SPLICE_RIGHT});
2074
2075 if (Subtarget.hasStdExtZvzip())
2076 setTargetDAGCombine({ISD::VECTOR_INTERLEAVE});
2077
2078 if (Subtarget.hasVendorXTHeadMemPair())
2079 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
2080 if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
2081 setTargetDAGCombine(ISD::BITCAST);
2082
2083 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2084
2085 setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2086
2087 setJumpIsExpensive(Subtarget.isJumpExpensive());
2088
2089 // Disable strict node mutation.
2090 IsStrictFPEnabled = true;
2091 EnableExtLdPromotion = true;
2092
2093 // Let the subtarget decide if a predictable select is more expensive than the
2094 // corresponding branch. This information is used in CGP/SelectOpt to decide
2095 // when to convert selects into branches.
2096 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
2097
2098 MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
2099 MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
2100
2101 MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
2102 MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
2103 MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
2104
2105 MaxStoresPerMemmoveOptSize =
2106 Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
2107 MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
2108
2109 MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
2110 MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
2111}
2112
2113TargetLoweringBase::LegalizeTypeAction
2114RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
2115 if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
2116 if (VT == MVT::v2i16 || VT == MVT::v4i8)
2117 return TypeWidenVector;
2118
2119 return TargetLoweringBase::getPreferredVectorAction(VT);
2120}
2121
2122EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
2123 LLVMContext &Context,
2124 EVT VT) const {
2125 if (!VT.isVector())
2126 return getPointerTy(DL);
2127 if (Subtarget.hasVInstructions() &&
2128 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
2129 return EVT::getVectorVT(Context, VT: MVT::i1, EC: VT.getVectorElementCount());
2130 return VT.changeVectorElementTypeToInteger();
2131}
2132
2133TargetLoweringBase::CondMergingParams
2134RISCVTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
2135 const Value *LHS,
2136 const Value *RHS,
2137 const Function *F) const {
2138 if (F->hasOptSize())
2139 return TargetLowering::getJumpConditionMergingParams(Opc, LHS, RHS, F);
2140
2141 // Merging conditions eliminates a branch, so the budget we are willing to
2142 // spend eagerly computing the RHS condition should scale with how expensive a
2143 // mispredicted branch is. A branch only costs the full penalty when actually
2144 // mispredicted, so scale it down by an assumed misprediction rate (~25%).
2145 int BaseCost = Subtarget.getMispredictionPenalty() / 4;
2146 if (BrMergingBaseCostThresh.getNumOccurrences() > 1)
2147 BaseCost = BrMergingBaseCostThresh;
2148
2149 return {.BaseCost: BaseCost, .LikelyBias: BrMergingLikelyBias, .UnlikelyBias: BrMergingUnlikelyBias};
2150}
2151
2152MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
2153 return Subtarget.getXLenVT();
2154}
2155
2156// Return false if we can lower get_vector_length to a vsetvli intrinsic.
2157bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
2158 unsigned VF,
2159 bool IsScalable) const {
2160 if (!Subtarget.hasVInstructions())
2161 return true;
2162
2163 if (!IsScalable)
2164 return true;
2165
2166 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
2167 return true;
2168
2169 // Don't allow VF=1 if those types are't legal.
2170 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
2171 return true;
2172
2173 // VLEN=32 support is incomplete.
2174 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
2175 return true;
2176
2177 // The maximum VF is for the smallest element width with LMUL=8.
2178 // VF must be a power of 2.
2179 unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
2180 return VF > MaxVF || !isPowerOf2_32(Value: VF);
2181}
2182
2183void RISCVTargetLowering::getTgtMemIntrinsic(
2184 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2185 MachineFunction &MF, unsigned Intrinsic) const {
2186 IntrinsicInfo Info;
2187 auto &DL = I.getDataLayout();
2188
2189 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
2190 bool IsUnitStrided, bool UsePtrVal = false) {
2191 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
2192 // We can't use ptrVal if the intrinsic can access memory before the
2193 // pointer. This means we can't use it for strided or indexed intrinsics.
2194 if (UsePtrVal)
2195 Info.ptrVal = I.getArgOperand(i: PtrOp);
2196 else
2197 Info.fallbackAddressSpace =
2198 I.getArgOperand(i: PtrOp)->getType()->getPointerAddressSpace();
2199 Type *MemTy;
2200 if (IsStore) {
2201 // Store value is the first operand.
2202 MemTy = I.getArgOperand(i: 0)->getType();
2203 } else {
2204 // Use return type. If it's segment load, return type is a struct.
2205 MemTy = I.getType();
2206 if (MemTy->isStructTy())
2207 MemTy = MemTy->getStructElementType(N: 0);
2208 }
2209 if (!IsUnitStrided)
2210 MemTy = MemTy->getScalarType();
2211
2212 Info.memVT = getValueType(DL, Ty: MemTy);
2213 if (MemTy->isTargetExtTy()) {
2214 // RISC-V vector tuple type's alignment type should be its element type.
2215 if (cast<TargetExtType>(Val: MemTy)->getName() == "riscv.vector.tuple")
2216 MemTy = Type::getIntNTy(
2217 C&: MemTy->getContext(),
2218 N: 1 << cast<ConstantInt>(Val: I.getArgOperand(i: I.arg_size() - 1))
2219 ->getZExtValue());
2220 Info.align = DL.getABITypeAlign(Ty: MemTy);
2221 } else {
2222 Info.align = Align(DL.getTypeStoreSize(Ty: MemTy->getScalarType()));
2223 }
2224 Info.size = MemoryLocation::UnknownSize;
2225 Info.flags |=
2226 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2227 Infos.push_back(Elt: Info);
2228 };
2229
2230 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
2231 Info.flags |= MachineMemOperand::MONonTemporal;
2232
2233 Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
2234 switch (Intrinsic) {
2235 default:
2236 return;
2237 case Intrinsic::riscv_masked_atomicrmw_xchg:
2238 case Intrinsic::riscv_masked_atomicrmw_add:
2239 case Intrinsic::riscv_masked_atomicrmw_sub:
2240 case Intrinsic::riscv_masked_atomicrmw_nand:
2241 case Intrinsic::riscv_masked_atomicrmw_max:
2242 case Intrinsic::riscv_masked_atomicrmw_min:
2243 case Intrinsic::riscv_masked_atomicrmw_umax:
2244 case Intrinsic::riscv_masked_atomicrmw_umin:
2245 case Intrinsic::riscv_masked_cmpxchg:
2246 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
2247 // narrow atomic operation. These will be expanded to an LR/SC loop that
2248 // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
2249 // will be used to modify the appropriate part of the 4 byte data and
2250 // preserve the rest.
2251 Info.opc = ISD::INTRINSIC_W_CHAIN;
2252 Info.memVT = MVT::i32;
2253 Info.ptrVal = I.getArgOperand(i: 0);
2254 Info.offset = 0;
2255 Info.align = Align(4);
2256 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2257 MachineMemOperand::MOVolatile;
2258 Infos.push_back(Elt: Info);
2259 return;
2260 case Intrinsic::riscv_seg2_load_mask:
2261 case Intrinsic::riscv_seg3_load_mask:
2262 case Intrinsic::riscv_seg4_load_mask:
2263 case Intrinsic::riscv_seg5_load_mask:
2264 case Intrinsic::riscv_seg6_load_mask:
2265 case Intrinsic::riscv_seg7_load_mask:
2266 case Intrinsic::riscv_seg8_load_mask:
2267 case Intrinsic::riscv_sseg2_load_mask:
2268 case Intrinsic::riscv_sseg3_load_mask:
2269 case Intrinsic::riscv_sseg4_load_mask:
2270 case Intrinsic::riscv_sseg5_load_mask:
2271 case Intrinsic::riscv_sseg6_load_mask:
2272 case Intrinsic::riscv_sseg7_load_mask:
2273 case Intrinsic::riscv_sseg8_load_mask:
2274 SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
2275 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2276 return;
2277 case Intrinsic::riscv_seg2_store_mask:
2278 case Intrinsic::riscv_seg3_store_mask:
2279 case Intrinsic::riscv_seg4_store_mask:
2280 case Intrinsic::riscv_seg5_store_mask:
2281 case Intrinsic::riscv_seg6_store_mask:
2282 case Intrinsic::riscv_seg7_store_mask:
2283 case Intrinsic::riscv_seg8_store_mask:
2284 // Operands are (vec, ..., vec, ptr, mask, vl)
2285 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2286 /*IsStore*/ true,
2287 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2288 return;
2289 case Intrinsic::riscv_sseg2_store_mask:
2290 case Intrinsic::riscv_sseg3_store_mask:
2291 case Intrinsic::riscv_sseg4_store_mask:
2292 case Intrinsic::riscv_sseg5_store_mask:
2293 case Intrinsic::riscv_sseg6_store_mask:
2294 case Intrinsic::riscv_sseg7_store_mask:
2295 case Intrinsic::riscv_sseg8_store_mask:
2296 // Operands are (vec, ..., vec, ptr, offset, mask, vl)
2297 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2298 /*IsStore*/ true,
2299 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2300 return;
2301 case Intrinsic::riscv_vlm:
2302 SetRVVLoadStoreInfo(/*PtrOp*/ 0,
2303 /*IsStore*/ false,
2304 /*IsUnitStrided*/ true,
2305 /*UsePtrVal*/ true);
2306 return;
2307 case Intrinsic::riscv_vle:
2308 case Intrinsic::riscv_vle_mask:
2309 case Intrinsic::riscv_vleff:
2310 case Intrinsic::riscv_vleff_mask:
2311 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2312 /*IsStore*/ false,
2313 /*IsUnitStrided*/ true,
2314 /*UsePtrVal*/ true);
2315 return;
2316 case Intrinsic::riscv_vsm:
2317 case Intrinsic::riscv_vse:
2318 case Intrinsic::riscv_vse_mask:
2319 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2320 /*IsStore*/ true,
2321 /*IsUnitStrided*/ true,
2322 /*UsePtrVal*/ true);
2323 return;
2324 case Intrinsic::riscv_vlse:
2325 case Intrinsic::riscv_vlse_mask:
2326 case Intrinsic::riscv_vloxei:
2327 case Intrinsic::riscv_vloxei_mask:
2328 case Intrinsic::riscv_vluxei:
2329 case Intrinsic::riscv_vluxei_mask:
2330 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2331 /*IsStore*/ false,
2332 /*IsUnitStrided*/ false);
2333 return;
2334 case Intrinsic::riscv_vsse:
2335 case Intrinsic::riscv_vsse_mask:
2336 case Intrinsic::riscv_vsoxei:
2337 case Intrinsic::riscv_vsoxei_mask:
2338 case Intrinsic::riscv_vsuxei:
2339 case Intrinsic::riscv_vsuxei_mask:
2340 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2341 /*IsStore*/ true,
2342 /*IsUnitStrided*/ false);
2343 return;
2344 case Intrinsic::riscv_vlseg2:
2345 case Intrinsic::riscv_vlseg3:
2346 case Intrinsic::riscv_vlseg4:
2347 case Intrinsic::riscv_vlseg5:
2348 case Intrinsic::riscv_vlseg6:
2349 case Intrinsic::riscv_vlseg7:
2350 case Intrinsic::riscv_vlseg8:
2351 case Intrinsic::riscv_vlseg2ff:
2352 case Intrinsic::riscv_vlseg3ff:
2353 case Intrinsic::riscv_vlseg4ff:
2354 case Intrinsic::riscv_vlseg5ff:
2355 case Intrinsic::riscv_vlseg6ff:
2356 case Intrinsic::riscv_vlseg7ff:
2357 case Intrinsic::riscv_vlseg8ff:
2358 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2359 /*IsStore*/ false,
2360 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2361 return;
2362 case Intrinsic::riscv_vlseg2_mask:
2363 case Intrinsic::riscv_vlseg3_mask:
2364 case Intrinsic::riscv_vlseg4_mask:
2365 case Intrinsic::riscv_vlseg5_mask:
2366 case Intrinsic::riscv_vlseg6_mask:
2367 case Intrinsic::riscv_vlseg7_mask:
2368 case Intrinsic::riscv_vlseg8_mask:
2369 case Intrinsic::riscv_vlseg2ff_mask:
2370 case Intrinsic::riscv_vlseg3ff_mask:
2371 case Intrinsic::riscv_vlseg4ff_mask:
2372 case Intrinsic::riscv_vlseg5ff_mask:
2373 case Intrinsic::riscv_vlseg6ff_mask:
2374 case Intrinsic::riscv_vlseg7ff_mask:
2375 case Intrinsic::riscv_vlseg8ff_mask:
2376 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2377 /*IsStore*/ false,
2378 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2379 return;
2380 case Intrinsic::riscv_vlsseg2:
2381 case Intrinsic::riscv_vlsseg3:
2382 case Intrinsic::riscv_vlsseg4:
2383 case Intrinsic::riscv_vlsseg5:
2384 case Intrinsic::riscv_vlsseg6:
2385 case Intrinsic::riscv_vlsseg7:
2386 case Intrinsic::riscv_vlsseg8:
2387 case Intrinsic::riscv_vloxseg2:
2388 case Intrinsic::riscv_vloxseg3:
2389 case Intrinsic::riscv_vloxseg4:
2390 case Intrinsic::riscv_vloxseg5:
2391 case Intrinsic::riscv_vloxseg6:
2392 case Intrinsic::riscv_vloxseg7:
2393 case Intrinsic::riscv_vloxseg8:
2394 case Intrinsic::riscv_vluxseg2:
2395 case Intrinsic::riscv_vluxseg3:
2396 case Intrinsic::riscv_vluxseg4:
2397 case Intrinsic::riscv_vluxseg5:
2398 case Intrinsic::riscv_vluxseg6:
2399 case Intrinsic::riscv_vluxseg7:
2400 case Intrinsic::riscv_vluxseg8:
2401 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2402 /*IsStore*/ false,
2403 /*IsUnitStrided*/ false);
2404 return;
2405 case Intrinsic::riscv_vlsseg2_mask:
2406 case Intrinsic::riscv_vlsseg3_mask:
2407 case Intrinsic::riscv_vlsseg4_mask:
2408 case Intrinsic::riscv_vlsseg5_mask:
2409 case Intrinsic::riscv_vlsseg6_mask:
2410 case Intrinsic::riscv_vlsseg7_mask:
2411 case Intrinsic::riscv_vlsseg8_mask:
2412 case Intrinsic::riscv_vloxseg2_mask:
2413 case Intrinsic::riscv_vloxseg3_mask:
2414 case Intrinsic::riscv_vloxseg4_mask:
2415 case Intrinsic::riscv_vloxseg5_mask:
2416 case Intrinsic::riscv_vloxseg6_mask:
2417 case Intrinsic::riscv_vloxseg7_mask:
2418 case Intrinsic::riscv_vloxseg8_mask:
2419 case Intrinsic::riscv_vluxseg2_mask:
2420 case Intrinsic::riscv_vluxseg3_mask:
2421 case Intrinsic::riscv_vluxseg4_mask:
2422 case Intrinsic::riscv_vluxseg5_mask:
2423 case Intrinsic::riscv_vluxseg6_mask:
2424 case Intrinsic::riscv_vluxseg7_mask:
2425 case Intrinsic::riscv_vluxseg8_mask:
2426 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
2427 /*IsStore*/ false,
2428 /*IsUnitStrided*/ false);
2429 return;
2430 case Intrinsic::riscv_vsseg2:
2431 case Intrinsic::riscv_vsseg3:
2432 case Intrinsic::riscv_vsseg4:
2433 case Intrinsic::riscv_vsseg5:
2434 case Intrinsic::riscv_vsseg6:
2435 case Intrinsic::riscv_vsseg7:
2436 case Intrinsic::riscv_vsseg8:
2437 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2438 /*IsStore*/ true,
2439 /*IsUnitStrided*/ false);
2440 return;
2441 case Intrinsic::riscv_vsseg2_mask:
2442 case Intrinsic::riscv_vsseg3_mask:
2443 case Intrinsic::riscv_vsseg4_mask:
2444 case Intrinsic::riscv_vsseg5_mask:
2445 case Intrinsic::riscv_vsseg6_mask:
2446 case Intrinsic::riscv_vsseg7_mask:
2447 case Intrinsic::riscv_vsseg8_mask:
2448 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2449 /*IsStore*/ true,
2450 /*IsUnitStrided*/ false);
2451 return;
2452 case Intrinsic::riscv_vssseg2:
2453 case Intrinsic::riscv_vssseg3:
2454 case Intrinsic::riscv_vssseg4:
2455 case Intrinsic::riscv_vssseg5:
2456 case Intrinsic::riscv_vssseg6:
2457 case Intrinsic::riscv_vssseg7:
2458 case Intrinsic::riscv_vssseg8:
2459 case Intrinsic::riscv_vsoxseg2:
2460 case Intrinsic::riscv_vsoxseg3:
2461 case Intrinsic::riscv_vsoxseg4:
2462 case Intrinsic::riscv_vsoxseg5:
2463 case Intrinsic::riscv_vsoxseg6:
2464 case Intrinsic::riscv_vsoxseg7:
2465 case Intrinsic::riscv_vsoxseg8:
2466 case Intrinsic::riscv_vsuxseg2:
2467 case Intrinsic::riscv_vsuxseg3:
2468 case Intrinsic::riscv_vsuxseg4:
2469 case Intrinsic::riscv_vsuxseg5:
2470 case Intrinsic::riscv_vsuxseg6:
2471 case Intrinsic::riscv_vsuxseg7:
2472 case Intrinsic::riscv_vsuxseg8:
2473 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2474 /*IsStore*/ true,
2475 /*IsUnitStrided*/ false);
2476 return;
2477 case Intrinsic::riscv_vssseg2_mask:
2478 case Intrinsic::riscv_vssseg3_mask:
2479 case Intrinsic::riscv_vssseg4_mask:
2480 case Intrinsic::riscv_vssseg5_mask:
2481 case Intrinsic::riscv_vssseg6_mask:
2482 case Intrinsic::riscv_vssseg7_mask:
2483 case Intrinsic::riscv_vssseg8_mask:
2484 case Intrinsic::riscv_vsoxseg2_mask:
2485 case Intrinsic::riscv_vsoxseg3_mask:
2486 case Intrinsic::riscv_vsoxseg4_mask:
2487 case Intrinsic::riscv_vsoxseg5_mask:
2488 case Intrinsic::riscv_vsoxseg6_mask:
2489 case Intrinsic::riscv_vsoxseg7_mask:
2490 case Intrinsic::riscv_vsoxseg8_mask:
2491 case Intrinsic::riscv_vsuxseg2_mask:
2492 case Intrinsic::riscv_vsuxseg3_mask:
2493 case Intrinsic::riscv_vsuxseg4_mask:
2494 case Intrinsic::riscv_vsuxseg5_mask:
2495 case Intrinsic::riscv_vsuxseg6_mask:
2496 case Intrinsic::riscv_vsuxseg7_mask:
2497 case Intrinsic::riscv_vsuxseg8_mask:
2498 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2499 /*IsStore*/ true,
2500 /*IsUnitStrided*/ false);
2501 return;
2502 case Intrinsic::riscv_sf_vlte8:
2503 case Intrinsic::riscv_sf_vlte16:
2504 case Intrinsic::riscv_sf_vlte32:
2505 case Intrinsic::riscv_sf_vlte64:
2506 Info.opc = ISD::INTRINSIC_VOID;
2507 Info.ptrVal = I.getArgOperand(i: 1);
2508 switch (Intrinsic) {
2509 case Intrinsic::riscv_sf_vlte8:
2510 Info.memVT = MVT::i8;
2511 Info.align = Align(1);
2512 break;
2513 case Intrinsic::riscv_sf_vlte16:
2514 Info.memVT = MVT::i16;
2515 Info.align = Align(2);
2516 break;
2517 case Intrinsic::riscv_sf_vlte32:
2518 Info.memVT = MVT::i32;
2519 Info.align = Align(4);
2520 break;
2521 case Intrinsic::riscv_sf_vlte64:
2522 Info.memVT = MVT::i64;
2523 Info.align = Align(8);
2524 break;
2525 }
2526 Info.size = MemoryLocation::UnknownSize;
2527 Info.flags |= MachineMemOperand::MOLoad;
2528 Infos.push_back(Elt: Info);
2529 return;
2530 case Intrinsic::riscv_sf_vste8:
2531 case Intrinsic::riscv_sf_vste16:
2532 case Intrinsic::riscv_sf_vste32:
2533 case Intrinsic::riscv_sf_vste64:
2534 Info.opc = ISD::INTRINSIC_VOID;
2535 Info.ptrVal = I.getArgOperand(i: 1);
2536 switch (Intrinsic) {
2537 case Intrinsic::riscv_sf_vste8:
2538 Info.memVT = MVT::i8;
2539 Info.align = Align(1);
2540 break;
2541 case Intrinsic::riscv_sf_vste16:
2542 Info.memVT = MVT::i16;
2543 Info.align = Align(2);
2544 break;
2545 case Intrinsic::riscv_sf_vste32:
2546 Info.memVT = MVT::i32;
2547 Info.align = Align(4);
2548 break;
2549 case Intrinsic::riscv_sf_vste64:
2550 Info.memVT = MVT::i64;
2551 Info.align = Align(8);
2552 break;
2553 }
2554 Info.size = MemoryLocation::UnknownSize;
2555 Info.flags |= MachineMemOperand::MOStore;
2556 Infos.push_back(Elt: Info);
2557 return;
2558 }
2559}
2560
2561bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2562 const AddrMode &AM, Type *Ty,
2563 unsigned AS,
2564 Instruction *I) const {
2565 // No global is ever allowed as a base.
2566 if (AM.BaseGV)
2567 return false;
2568
2569 // None of our addressing modes allows a scalable offset
2570 if (AM.ScalableOffset)
2571 return false;
2572
2573 // RVV instructions only support register addressing.
2574 if (Subtarget.hasVInstructions() && isa<VectorType>(Val: Ty))
2575 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
2576
2577 // The Xqcilo extension provides load/store instructions with a 26-bit signed
2578 // offset.
2579 if (Subtarget.hasVendorXqcilo()) {
2580 if (!isInt<26>(x: AM.BaseOffs))
2581 return false;
2582 } else if (!isInt<12>(x: AM.BaseOffs)) {
2583 // Otherwise require a 12-bit signed offset.
2584 return false;
2585 }
2586
2587 switch (AM.Scale) {
2588 case 0: // "r+i" or just "i", depending on HasBaseReg.
2589 break;
2590 case 1:
2591 if (!AM.HasBaseReg) // allow "r+i".
2592 break;
2593 return false; // disallow "r+r" or "r+r+i".
2594 default:
2595 return false;
2596 }
2597
2598 return true;
2599}
2600
2601bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
2602 return isInt<12>(x: Imm);
2603}
2604
2605bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
2606 // The Xqcilia extension provides add-immediate instructions with a 26-bit
2607 // signed immediate.
2608 if (Subtarget.hasVendorXqcilia())
2609 return isInt<26>(x: Imm);
2610 return isInt<12>(x: Imm);
2611}
2612
2613// On RV32, 64-bit integers are split into their high and low parts and held
2614// in two different registers, so the trunc is free since the low register can
2615// just be used.
2616// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
2617// isTruncateFree?
2618bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
2619 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
2620 return false;
2621 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
2622 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
2623 return (SrcBits == 64 && DestBits == 32);
2624}
2625
2626bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
2627 // We consider i64->i32 free on RV64 since we have good selection of W
2628 // instructions that make promoting operations back to i64 free in many cases.
2629 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
2630 !DstVT.isInteger())
2631 return false;
2632 unsigned SrcBits = SrcVT.getSizeInBits();
2633 unsigned DestBits = DstVT.getSizeInBits();
2634 return (SrcBits == 64 && DestBits == 32);
2635}
2636
2637bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
2638 EVT SrcVT = Val.getValueType();
2639 // free truncate from vnsrl and vnsra
2640 if (Subtarget.hasVInstructions() &&
2641 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
2642 SrcVT.isVector() && VT2.isVector()) {
2643 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
2644 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
2645 if (SrcBits == DestBits * 2) {
2646 return true;
2647 }
2648 }
2649 return TargetLowering::isTruncateFree(Val, VT2);
2650}
2651
2652bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
2653 // Zexts are free if they can be combined with a load.
2654 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
2655 // poorly with type legalization of compares preferring sext.
2656 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
2657 EVT MemVT = LD->getMemoryVT();
2658 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
2659 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
2660 LD->getExtensionType() == ISD::ZEXTLOAD))
2661 return true;
2662 }
2663
2664 return TargetLowering::isZExtFree(Val, VT2);
2665}
2666
2667bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
2668 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
2669}
2670
2671bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
2672 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(BitWidth: 32);
2673}
2674
2675bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
2676 return Subtarget.hasCTZLike();
2677}
2678
2679bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
2680 return Subtarget.hasCLZLike();
2681}
2682
2683bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
2684 const Instruction &AndI) const {
2685 // We expect to be able to match a bit extraction instruction if the Zbs
2686 // extension is supported and the mask is a power of two. However, we
2687 // conservatively return false if the mask would fit in an ANDI instruction,
2688 // on the basis that it's possible the sinking+duplication of the AND in
2689 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
2690 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
2691 if (!Subtarget.hasBEXTILike())
2692 return false;
2693 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
2694 if (!Mask)
2695 return false;
2696 return !Mask->getValue().isSignedIntN(N: 12) && Mask->getValue().isPowerOf2();
2697}
2698
2699bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
2700 EVT VT = Y.getValueType();
2701
2702 if (VT.isVector())
2703 return false;
2704
2705 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2706 (!isa<ConstantSDNode>(Val: Y) || cast<ConstantSDNode>(Val&: Y)->isOpaque());
2707}
2708
2709bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
2710 EVT VT = Y.getValueType();
2711
2712 if (!VT.isVector())
2713 return hasAndNotCompare(Y);
2714
2715 // vmandn.mm
2716 if (VT.getVectorElementType() == MVT::i1)
2717 return Subtarget.hasVInstructions();
2718
2719 return Subtarget.hasStdExtZvkb();
2720}
2721
2722bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2723 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
2724 if (Subtarget.hasStdExtZbs())
2725 return X.getValueType().isScalarInteger();
2726 auto *C = dyn_cast<ConstantSDNode>(Val&: Y);
2727 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
2728 if (Subtarget.hasVendorXTHeadBs())
2729 return C != nullptr;
2730 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
2731 return C && C->getAPIntValue().ule(RHS: 10);
2732}
2733
2734bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
2735 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
2736 SDValue Y) const {
2737 if (SelectOpcode != ISD::VSELECT)
2738 return false;
2739
2740 // Only enable for rvv.
2741 if (!VT.isVector() || !Subtarget.hasVInstructions())
2742 return false;
2743
2744 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2745 return false;
2746
2747 return true;
2748}
2749
2750bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2751 Type *Ty) const {
2752 assert(Ty->isIntegerTy());
2753
2754 unsigned BitSize = Ty->getIntegerBitWidth();
2755 if (BitSize > Subtarget.getXLen())
2756 return false;
2757
2758 // Fast path, assume 32-bit immediates are cheap.
2759 int64_t Val = Imm.getSExtValue();
2760 if (isInt<32>(x: Val))
2761 return true;
2762
2763 // A constant pool entry may be more aligned than the load we're trying to
2764 // replace. If we don't support unaligned scalar mem, prefer the constant
2765 // pool.
2766 // TODO: Can the caller pass down the alignment?
2767 if (!Subtarget.enableUnalignedScalarMem())
2768 return true;
2769
2770 // Prefer to keep the load if it would require many instructions.
2771 // This uses the same threshold we use for constant pools but doesn't
2772 // check useConstantPoolForLargeInts.
2773 // TODO: Should we keep the load only when we're definitely going to emit a
2774 // constant pool?
2775
2776 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI: Subtarget);
2777 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
2778}
2779
2780bool RISCVTargetLowering::
2781 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2782 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
2783 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
2784 SelectionDAG &DAG) const {
2785 // One interesting pattern that we'd want to form is 'bit extract':
2786 // ((1 >> Y) & 1) ==/!= 0
2787 // But we also need to be careful not to try to reverse that fold.
2788
2789 // Is this '((1 >> Y) & 1)'?
2790 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2791 return false; // Keep the 'bit extract' pattern.
2792
2793 // Will this be '((1 >> Y) & 1)' after the transform?
2794 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2795 return true; // Do form the 'bit extract' pattern.
2796
2797 // If 'X' is a constant, and we transform, then we will immediately
2798 // try to undo the fold, thus causing endless combine loop.
2799 // So only do the transform if X is not a constant. This matches the default
2800 // implementation of this function.
2801 return !XC;
2802}
2803
2804bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
2805 unsigned Opc = VecOp.getOpcode();
2806
2807 // Assume target opcodes can't be scalarized.
2808 // TODO - do we have any exceptions?
2809 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
2810 return false;
2811
2812 // If the vector op is not supported, try to convert to scalar.
2813 EVT VecVT = VecOp.getValueType();
2814 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
2815 return true;
2816
2817 // If the vector op is supported, but the scalar op is not, the transform may
2818 // not be worthwhile.
2819 // Permit a vector binary operation can be converted to scalar binary
2820 // operation which is custom lowered with illegal type.
2821 EVT ScalarVT = VecVT.getScalarType();
2822 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT) ||
2823 isOperationCustom(Op: Opc, VT: ScalarVT);
2824}
2825
2826bool RISCVTargetLowering::isOffsetFoldingLegal(
2827 const GlobalAddressSDNode *GA) const {
2828 // In order to maximise the opportunity for common subexpression elimination,
2829 // keep a separate ADD node for the global address offset instead of folding
2830 // it in the global address node. Later peephole optimisations may choose to
2831 // fold it back in when profitable.
2832 return false;
2833}
2834
2835// Returns 0-31 if the fli instruction is available for the type and this is
2836// legal FP immediate for the type. Returns -1 otherwise.
2837int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2838 if (!Subtarget.hasStdExtZfa())
2839 return -1;
2840
2841 bool IsSupportedVT = false;
2842 if (VT == MVT::f16) {
2843 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2844 } else if (VT == MVT::f32) {
2845 IsSupportedVT = true;
2846 } else if (VT == MVT::f64) {
2847 assert(Subtarget.hasStdExtD() && "Expect D extension");
2848 IsSupportedVT = true;
2849 }
2850
2851 if (!IsSupportedVT)
2852 return -1;
2853
2854 return RISCVLoadFPImm::getLoadFPImm(FPImm: Imm);
2855}
2856
2857bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2858 bool ForCodeSize) const {
2859 bool IsLegalVT = false;
2860 if (VT == MVT::f16)
2861 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2862 else if (VT == MVT::f32)
2863 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2864 else if (VT == MVT::f64)
2865 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2866 else if (VT == MVT::bf16)
2867 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2868
2869 if (!IsLegalVT)
2870 return false;
2871
2872 if (getLegalZfaFPImm(Imm, VT) >= 0)
2873 return true;
2874
2875 // Some constants can be produced by fli+fneg.
2876 if (Imm.isNegative() && getLegalZfaFPImm(Imm: -Imm, VT) >= 0)
2877 return true;
2878
2879 // Cannot create a 64 bit floating-point immediate value for rv32.
2880 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2881 // td can handle +0.0 or -0.0 already.
2882 // -0.0 can be created by fmv + fneg.
2883 return Imm.isZero();
2884 }
2885
2886 // Special case: fmv + fneg
2887 if (Imm.isNegZero())
2888 return true;
2889
2890 // Building an integer and then converting requires a fmv at the end of
2891 // the integer sequence. The fmv is not required for Zfinx.
2892 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2893 const int Cost =
2894 FmvCost + RISCVMatInt::getIntMatCost(Val: Imm.bitcastToAPInt(),
2895 Size: Subtarget.getXLen(), STI: Subtarget);
2896 return Cost <= FPImmCost;
2897}
2898
2899// TODO: This is very conservative.
2900TargetLowering::ExtractSubvectorCost
2901RISCVTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
2902 unsigned Index) const {
2903 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
2904 (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) {
2905 if ((Index % ResVT.getVectorNumElements()) == 0)
2906 return ExtractSubvectorCost::Free;
2907 return ExtractSubvectorCost::Expensive;
2908 }
2909
2910 if (!Subtarget.hasVInstructions())
2911 return ExtractSubvectorCost::Expensive;
2912
2913 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2914 return ExtractSubvectorCost::Expensive;
2915
2916 // Extracts from index 0 are just subreg extracts.
2917 if (Index == 0)
2918 return ExtractSubvectorCost::Free;
2919
2920 // Only support extracting a fixed from a fixed vector for now.
2921 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2922 return ExtractSubvectorCost::Expensive;
2923
2924 EVT EltVT = ResVT.getVectorElementType();
2925 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2926
2927 // The smallest type we can slide is i8.
2928 if (EltVT == MVT::i1)
2929 return ExtractSubvectorCost::Expensive;
2930
2931 unsigned ResElts = ResVT.getVectorNumElements();
2932 unsigned SrcElts = SrcVT.getVectorNumElements();
2933
2934 unsigned MinVLen = Subtarget.getRealMinVLen();
2935 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2936
2937 // If we're extracting only data from the first VLEN bits of the source
2938 // then we can always do this with an m1 vslidedown.vx. Restricting the
2939 // Index ensures we can use a vslidedown.vi.
2940 // TODO: We can generalize this when the exact VLEN is known.
2941 if (Index + ResElts <= MinVLMAX && Index < 31)
2942 return ExtractSubvectorCost::Free;
2943
2944 // Convervatively only handle extracting half of a vector.
2945 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2946 // the upper half of a vector until we have more test coverage.
2947 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2948 // a cheap extract. However, this case is important in practice for
2949 // shuffled extracts of longer vectors. How resolve?
2950 if ((ResElts * 2) == SrcElts && Index == ResElts)
2951 return ExtractSubvectorCost::Free;
2952 return ExtractSubvectorCost::Expensive;
2953}
2954
2955MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2956 CallingConv::ID CC,
2957 EVT VT) const {
2958 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2959 // We might still end up using a GPR but that will be decided based on ABI.
2960 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2961 !Subtarget.hasStdExtZfhminOrZhinxmin())
2962 return MVT::f32;
2963
2964 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2965}
2966
2967unsigned
2968RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2969 std::optional<MVT> RegisterVT) const {
2970 // Pair inline assembly operand
2971 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2972 *RegisterVT == MVT::Untyped)
2973 return 1;
2974
2975 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2976}
2977
2978unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2979 CallingConv::ID CC,
2980 EVT VT) const {
2981 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2982 // We might still end up using a GPR but that will be decided based on ABI.
2983 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2984 !Subtarget.hasStdExtZfhminOrZhinxmin())
2985 return 1;
2986
2987 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2988}
2989
2990// Changes the condition code and swaps operands if necessary, so the SetCC
2991// operation matches one of the comparisons supported directly by branches
2992// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2993// with 1/-1.
2994static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2995 ISD::CondCode &CC, SelectionDAG &DAG,
2996 const RISCVSubtarget &Subtarget) {
2997 // If this is a single bit test that can't be handled by ANDI, shift the
2998 // bit to be tested to the MSB and perform a signed compare with 0.
2999 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
3000 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
3001 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
3002 // XAndesPerf supports branch on test bit.
3003 !Subtarget.hasVendorXAndesPerf()) {
3004 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
3005 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
3006 unsigned ShAmt = 0;
3007 if (isPowerOf2_64(Value: Mask)) {
3008 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
3009 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
3010 } else {
3011 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
3012 }
3013
3014 LHS = LHS.getOperand(i: 0);
3015 if (ShAmt != 0)
3016 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
3017 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
3018 return;
3019 }
3020 }
3021
3022 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3023 int64_t C = RHSC->getSExtValue();
3024 switch (CC) {
3025 default: break;
3026 case ISD::SETGT:
3027 // Convert X > -1 to X >= 0.
3028 if (C == -1) {
3029 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
3030 CC = ISD::SETGE;
3031 return;
3032 }
3033 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3034 C != INT64_MAX && isInt<5>(x: C + 1)) {
3035 // We have a conditional move instruction for SETGE but not SETGT.
3036 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3037 RHS = DAG.getSignedConstant(Val: C + 1, DL, VT: RHS.getValueType());
3038 CC = ISD::SETGE;
3039 return;
3040 }
3041 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(x: C + 1)) {
3042 // We have a branch immediate instruction for SETGE but not SETGT.
3043 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
3044 RHS = DAG.getSignedConstant(Val: C + 1, DL, VT: RHS.getValueType());
3045 CC = ISD::SETGE;
3046 return;
3047 }
3048 break;
3049 case ISD::SETLT:
3050 // Convert X < 1 to 0 >= X.
3051 if (C == 1) {
3052 RHS = LHS;
3053 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
3054 CC = ISD::SETGE;
3055 return;
3056 }
3057 break;
3058 case ISD::SETUGT:
3059 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3060 C != INT64_MAX && isUInt<5>(x: C + 1)) {
3061 // We have a conditional move instruction for SETUGE but not SETUGT.
3062 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3063 RHS = DAG.getConstant(Val: C + 1, DL, VT: RHS.getValueType());
3064 CC = ISD::SETUGE;
3065 return;
3066 }
3067 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(x: C + 1)) {
3068 // We have a branch immediate instruction for SETUGE but not SETUGT.
3069 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
3070 // immediate.
3071 RHS = DAG.getConstant(Val: C + 1, DL, VT: RHS.getValueType());
3072 CC = ISD::SETUGE;
3073 return;
3074 }
3075 break;
3076 }
3077 }
3078
3079 switch (CC) {
3080 default:
3081 break;
3082 case ISD::SETGT:
3083 case ISD::SETLE:
3084 case ISD::SETUGT:
3085 case ISD::SETULE:
3086 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3087 std::swap(a&: LHS, b&: RHS);
3088 break;
3089 }
3090}
3091
3092RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
3093 if (VT.isRISCVVectorTuple()) {
3094 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
3095 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
3096 return RISCVVType::LMUL_F8;
3097 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
3098 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
3099 return RISCVVType::LMUL_F4;
3100 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
3101 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
3102 return RISCVVType::LMUL_F2;
3103 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
3104 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
3105 return RISCVVType::LMUL_1;
3106 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
3107 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
3108 return RISCVVType::LMUL_2;
3109 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
3110 return RISCVVType::LMUL_4;
3111 llvm_unreachable("Invalid vector tuple type LMUL.");
3112 }
3113
3114 assert(VT.isScalableVector() && "Expecting a scalable vector type");
3115 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
3116 if (VT.getVectorElementType() == MVT::i1)
3117 KnownSize *= 8;
3118
3119 switch (KnownSize) {
3120 default:
3121 llvm_unreachable("Invalid LMUL.");
3122 case 8:
3123 return RISCVVType::LMUL_F8;
3124 case 16:
3125 return RISCVVType::LMUL_F4;
3126 case 32:
3127 return RISCVVType::LMUL_F2;
3128 case 64:
3129 return RISCVVType::LMUL_1;
3130 case 128:
3131 return RISCVVType::LMUL_2;
3132 case 256:
3133 return RISCVVType::LMUL_4;
3134 case 512:
3135 return RISCVVType::LMUL_8;
3136 }
3137}
3138
3139unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
3140 switch (LMul) {
3141 default:
3142 llvm_unreachable("Invalid LMUL.");
3143 case RISCVVType::LMUL_F8:
3144 case RISCVVType::LMUL_F4:
3145 case RISCVVType::LMUL_F2:
3146 case RISCVVType::LMUL_1:
3147 return RISCV::VRRegClassID;
3148 case RISCVVType::LMUL_2:
3149 return RISCV::VRM2RegClassID;
3150 case RISCVVType::LMUL_4:
3151 return RISCV::VRM4RegClassID;
3152 case RISCVVType::LMUL_8:
3153 return RISCV::VRM8RegClassID;
3154 }
3155}
3156
3157unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
3158 RISCVVType::VLMUL LMUL = getLMUL(VT);
3159 if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
3160 LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
3161 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
3162 "Unexpected subreg numbering");
3163 return RISCV::sub_vrm1_0 + Index;
3164 }
3165 if (LMUL == RISCVVType::LMUL_2) {
3166 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
3167 "Unexpected subreg numbering");
3168 return RISCV::sub_vrm2_0 + Index;
3169 }
3170 if (LMUL == RISCVVType::LMUL_4) {
3171 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
3172 "Unexpected subreg numbering");
3173 return RISCV::sub_vrm4_0 + Index;
3174 }
3175 llvm_unreachable("Invalid vector type.");
3176}
3177
3178unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
3179 if (VT.isRISCVVectorTuple()) {
3180 unsigned NF = VT.getRISCVVectorTupleNumFields();
3181 unsigned RegsPerField =
3182 std::max(a: 1U, b: (unsigned)VT.getSizeInBits().getKnownMinValue() /
3183 (NF * RISCV::RVVBitsPerBlock));
3184 switch (RegsPerField) {
3185 case 1:
3186 if (NF == 2)
3187 return RISCV::VRN2M1RegClassID;
3188 if (NF == 3)
3189 return RISCV::VRN3M1RegClassID;
3190 if (NF == 4)
3191 return RISCV::VRN4M1RegClassID;
3192 if (NF == 5)
3193 return RISCV::VRN5M1RegClassID;
3194 if (NF == 6)
3195 return RISCV::VRN6M1RegClassID;
3196 if (NF == 7)
3197 return RISCV::VRN7M1RegClassID;
3198 if (NF == 8)
3199 return RISCV::VRN8M1RegClassID;
3200 break;
3201 case 2:
3202 if (NF == 2)
3203 return RISCV::VRN2M2RegClassID;
3204 if (NF == 3)
3205 return RISCV::VRN3M2RegClassID;
3206 if (NF == 4)
3207 return RISCV::VRN4M2RegClassID;
3208 break;
3209 case 4:
3210 assert(NF == 2);
3211 return RISCV::VRN2M4RegClassID;
3212 default:
3213 break;
3214 }
3215 llvm_unreachable("Invalid vector tuple type RegClass.");
3216 }
3217
3218 if (VT.getVectorElementType() == MVT::i1)
3219 return RISCV::VRRegClassID;
3220 return getRegClassIDForLMUL(LMul: getLMUL(VT));
3221}
3222
3223// Attempt to decompose a subvector insert/extract between VecVT and
3224// SubVecVT via subregister indices. Returns the subregister index that
3225// can perform the subvector insert/extract with the given element index, as
3226// well as the index corresponding to any leftover subvectors that must be
3227// further inserted/extracted within the register class for SubVecVT.
3228std::pair<unsigned, unsigned>
3229RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3230 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
3231 const RISCVRegisterInfo *TRI) {
3232 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
3233 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
3234 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
3235 "Register classes not ordered");
3236 unsigned VecRegClassID = getRegClassIDForVecVT(VT: VecVT);
3237 unsigned SubRegClassID = getRegClassIDForVecVT(VT: SubVecVT);
3238
3239 // If VecVT is a vector tuple type, either it's the tuple type with same
3240 // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
3241 if (VecVT.isRISCVVectorTuple()) {
3242 if (VecRegClassID == SubRegClassID)
3243 return {RISCV::NoSubRegister, 0};
3244
3245 assert(SubVecVT.isScalableVector() &&
3246 "Only allow scalable vector subvector.");
3247 assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
3248 "Invalid vector tuple insert/extract for vector and subvector with "
3249 "different LMUL.");
3250 return {getSubregIndexByMVT(VT: VecVT, Index: InsertExtractIdx), 0};
3251 }
3252
3253 // Try to compose a subregister index that takes us from the incoming
3254 // LMUL>1 register class down to the outgoing one. At each step we half
3255 // the LMUL:
3256 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
3257 // Note that this is not guaranteed to find a subregister index, such as
3258 // when we are extracting from one VR type to another.
3259 unsigned SubRegIdx = RISCV::NoSubRegister;
3260 for (const unsigned RCID :
3261 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
3262 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
3263 VecVT = VecVT.getHalfNumVectorElementsVT();
3264 bool IsHi =
3265 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
3266 SubRegIdx = TRI->composeSubRegIndices(a: SubRegIdx,
3267 b: getSubregIndexByMVT(VT: VecVT, Index: IsHi));
3268 if (IsHi)
3269 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
3270 }
3271 return {SubRegIdx, InsertExtractIdx};
3272}
3273
3274// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
3275// stores for those types.
3276bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
3277 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
3278 (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
3279 return false;
3280
3281 return !Subtarget.useRVVForFixedLengthVectors() ||
3282 VT.isFixedLengthVectorOf(EltVT: MVT::i1);
3283}
3284
3285bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
3286 if (!ScalarTy.isSimple())
3287 return false;
3288 switch (ScalarTy.getSimpleVT().SimpleTy) {
3289 case MVT::iPTR:
3290 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
3291 case MVT::i8:
3292 case MVT::i16:
3293 case MVT::i32:
3294 return Subtarget.hasVInstructions();
3295 case MVT::i64:
3296 return Subtarget.hasVInstructionsI64();
3297 case MVT::f16:
3298 return Subtarget.hasVInstructionsF16Minimal();
3299 case MVT::bf16:
3300 return Subtarget.hasVInstructionsBF16Minimal();
3301 case MVT::f32:
3302 return Subtarget.hasVInstructionsF32();
3303 case MVT::f64:
3304 return Subtarget.hasVInstructionsF64();
3305 default:
3306 return false;
3307 }
3308}
3309
3310
3311unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
3312 return NumRepeatedDivisors;
3313}
3314
3315static SDValue getVLOperand(SDValue Op) {
3316 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3317 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3318 "Unexpected opcode");
3319 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3320 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
3321 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3322 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
3323 if (!II)
3324 return SDValue();
3325 return Op.getOperand(i: II->VLOperand + 1 + HasChain);
3326}
3327
3328static bool useRVVForFixedLengthVectorVT(MVT VT,
3329 const RISCVSubtarget &Subtarget) {
3330 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
3331 if (!Subtarget.useRVVForFixedLengthVectors())
3332 return false;
3333
3334 // We only support a set of vector types with a consistent maximum fixed size
3335 // across all supported vector element types to avoid legalization issues.
3336 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
3337 // fixed-length vector type we support is 1024 bytes.
3338 if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
3339 return false;
3340
3341 unsigned MinVLen = Subtarget.getRealMinVLen();
3342
3343 MVT EltVT = VT.getVectorElementType();
3344
3345 // Don't use RVV for vectors we cannot scalarize if required.
3346 switch (EltVT.SimpleTy) {
3347 // i1 is supported but has different rules.
3348 default:
3349 return false;
3350 case MVT::i1:
3351 // Masks can only use a single register.
3352 if (VT.getVectorNumElements() > MinVLen)
3353 return false;
3354 MinVLen /= 8;
3355 break;
3356 case MVT::i8:
3357 case MVT::i16:
3358 case MVT::i32:
3359 break;
3360 case MVT::i64:
3361 if (!Subtarget.hasVInstructionsI64())
3362 return false;
3363 break;
3364 case MVT::f16:
3365 if (!Subtarget.hasVInstructionsF16Minimal())
3366 return false;
3367 break;
3368 case MVT::bf16:
3369 if (!Subtarget.hasVInstructionsBF16Minimal())
3370 return false;
3371 break;
3372 case MVT::f32:
3373 if (!Subtarget.hasVInstructionsF32())
3374 return false;
3375 break;
3376 case MVT::f64:
3377 if (!Subtarget.hasVInstructionsF64())
3378 return false;
3379 break;
3380 }
3381
3382 // Reject elements larger than ELEN.
3383 if (EltVT.getSizeInBits() > Subtarget.getELen())
3384 return false;
3385
3386 unsigned LMul = divideCeil(Numerator: VT.getSizeInBits(), Denominator: MinVLen);
3387 // Don't use RVV for types that don't fit.
3388 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
3389 return false;
3390
3391 // TODO: Perhaps an artificial restriction, but worth having whilst getting
3392 // the base fixed length RVV support in place.
3393 if (!VT.isPow2VectorType())
3394 return false;
3395
3396 return true;
3397}
3398
3399bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
3400 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
3401}
3402
3403// Return the largest legal scalable vector type that matches VT's element type.
3404static MVT getContainerForFixedLengthVector(MVT VT,
3405 const RISCVSubtarget &Subtarget) {
3406 // This may be called before legal types are setup.
3407 assert(((VT.isFixedLengthVector() &&
3408 Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
3409 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
3410 "Expected legal fixed length vector!");
3411
3412 unsigned MinVLen = Subtarget.getRealMinVLen();
3413 unsigned MaxELen = Subtarget.getELen();
3414
3415 MVT EltVT = VT.getVectorElementType();
3416 switch (EltVT.SimpleTy) {
3417 default:
3418 llvm_unreachable("unexpected element type for RVV container");
3419 case MVT::i1:
3420 case MVT::i8:
3421 case MVT::i16:
3422 case MVT::i32:
3423 case MVT::i64:
3424 case MVT::bf16:
3425 case MVT::f16:
3426 case MVT::f32:
3427 case MVT::f64: {
3428 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
3429 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
3430 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
3431 unsigned NumElts =
3432 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
3433 NumElts = std::max(a: NumElts, b: RISCV::RVVBitsPerBlock / MaxELen);
3434 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
3435 return MVT::getScalableVectorVT(VT: EltVT, NumElements: NumElts);
3436 }
3437 }
3438}
3439
3440MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
3441 return ::getContainerForFixedLengthVector(VT, Subtarget: getSubtarget());
3442}
3443
3444// Grow V to consume an entire RVV register.
3445static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3446 const RISCVSubtarget &Subtarget) {
3447 assert(VT.isScalableVector() &&
3448 "Expected to convert into a scalable vector!");
3449 assert(V.getValueType().isFixedLengthVector() &&
3450 "Expected a fixed length vector operand!");
3451 SDLoc DL(V);
3452 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: V, Idx: 0);
3453}
3454
3455// Shrink V so it's just big enough to maintain a VT's worth of data.
3456static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3457 const RISCVSubtarget &Subtarget) {
3458 assert(VT.isFixedLengthVector() &&
3459 "Expected to convert into a fixed length vector!");
3460 assert(V.getValueType().isScalableVector() &&
3461 "Expected a scalable vector operand!");
3462 SDLoc DL(V);
3463 return DAG.getExtractSubvector(DL, VT, Vec: V, Idx: 0);
3464}
3465
3466/// Return the type of the mask type suitable for masking the provided
3467/// vector type. This is simply an i1 element type vector of the same
3468/// (possibly scalable) length.
3469static MVT getMaskTypeFor(MVT VecVT) {
3470 assert(VecVT.isVector());
3471 ElementCount EC = VecVT.getVectorElementCount();
3472 return MVT::getVectorVT(VT: MVT::i1, EC);
3473}
3474
3475/// Creates an all ones mask suitable for masking a vector of type VecTy with
3476/// vector length VL. .
3477static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
3478 SelectionDAG &DAG) {
3479 MVT MaskVT = getMaskTypeFor(VecVT);
3480 return DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: MaskVT, Operand: VL);
3481}
3482
3483static std::pair<SDValue, SDValue>
3484getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
3485 const RISCVSubtarget &Subtarget) {
3486 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
3487 SDValue VL = DAG.getRegister(Reg: RISCV::X0, VT: Subtarget.getXLenVT());
3488 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
3489 return {Mask, VL};
3490}
3491
3492static std::pair<SDValue, SDValue>
3493getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
3494 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
3495 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3496 SDValue VL = DAG.getConstant(Val: NumElts, DL, VT: Subtarget.getXLenVT());
3497 SDValue Mask = getAllOnesMask(VecVT: ContainerVT, VL, DL, DAG);
3498 return {Mask, VL};
3499}
3500
3501// Gets the two common "VL" operands: an all-ones mask and the vector length.
3502// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
3503// the vector type that the fixed-length vector is contained in. Otherwise if
3504// VecVT is scalable, then ContainerVT should be the same as VecVT.
3505static std::pair<SDValue, SDValue>
3506getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
3507 const RISCVSubtarget &Subtarget) {
3508 if (VecVT.isFixedLengthVector())
3509 return getDefaultVLOps(NumElts: VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
3510 Subtarget);
3511 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3512 return getDefaultScalableVLOps(VecVT: ContainerVT, DL, DAG, Subtarget);
3513}
3514
3515SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
3516 SelectionDAG &DAG) const {
3517 assert(VecVT.isScalableVector() && "Expected scalable vector");
3518 return DAG.getElementCount(DL, VT: Subtarget.getXLenVT(),
3519 EC: VecVT.getVectorElementCount());
3520}
3521
3522std::pair<unsigned, unsigned>
3523RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
3524 const RISCVSubtarget &Subtarget) {
3525 assert(VecVT.isScalableVector() && "Expected scalable vector");
3526
3527 unsigned EltSize = VecVT.getScalarSizeInBits();
3528 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
3529
3530 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
3531 unsigned MaxVLMAX =
3532 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
3533
3534 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
3535 unsigned MinVLMAX =
3536 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMin, EltSize, MinSize);
3537
3538 return std::make_pair(x&: MinVLMAX, y&: MaxVLMAX);
3539}
3540
3541// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
3542// of either is (currently) supported. This can get us into an infinite loop
3543// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
3544// as a ..., etc.
3545// Until either (or both) of these can reliably lower any node, reporting that
3546// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
3547// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
3548// which is not desirable.
3549bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
3550 EVT VT, unsigned DefinedValues) const {
3551 return false;
3552}
3553
3554InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
3555 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
3556 // implementation-defined.
3557 if (!VT.isVector())
3558 return InstructionCost::getInvalid();
3559 unsigned DLenFactor = Subtarget.getDLenFactor();
3560 unsigned Cost;
3561 if (VT.isScalableVector()) {
3562 unsigned LMul;
3563 bool Fractional;
3564 std::tie(args&: LMul, args&: Fractional) =
3565 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT));
3566 if (Fractional)
3567 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
3568 else
3569 Cost = (LMul * DLenFactor);
3570 } else {
3571 Cost = divideCeil(Numerator: VT.getSizeInBits(), Denominator: Subtarget.getRealMinVLen() / DLenFactor);
3572 }
3573 return Cost;
3574}
3575
3576
3577/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
3578/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
3579/// be by default. VRGatherCostModel reflects available options. Note that
3580/// operand (index and possibly mask) are handled separately.
3581InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
3582 auto LMULCost = getLMULCost(VT);
3583 bool Log2CostModel =
3584 Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
3585 if (Log2CostModel && LMULCost.isValid()) {
3586 unsigned Log = Log2_64(Value: LMULCost.getValue());
3587 if (Log > 0)
3588 return LMULCost * Log;
3589 }
3590 return LMULCost * LMULCost;
3591}
3592
3593/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
3594/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
3595/// or may track the vrgather.vv cost. It is implementation-dependent.
3596InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const {
3597 return getLMULCost(VT);
3598}
3599
3600/// Return the cost of a vslidedown.vx or vslideup.vx instruction
3601/// for the type VT. (This does not cover the vslide1up or vslide1down
3602/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3603/// or may track the vrgather.vv cost. It is implementation-dependent.
3604InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const {
3605 return getLMULCost(VT);
3606}
3607
3608/// Return the cost of a vslidedown.vi or vslideup.vi instruction
3609/// for the type VT. (This does not cover the vslide1up or vslide1down
3610/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3611/// or may track the vrgather.vv cost. It is implementation-dependent.
3612InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const {
3613 return getLMULCost(VT);
3614}
3615
3616static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
3617 const RISCVSubtarget &Subtarget) {
3618 // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
3619 // bf16 conversions are always promoted to f32.
3620 if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3621 Op.getValueType() == MVT::bf16) {
3622 bool IsStrict = Op->isStrictFPOpcode();
3623
3624 SDLoc DL(Op);
3625 if (IsStrict) {
3626 SDValue Val = DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {MVT::f32, MVT::Other},
3627 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
3628 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
3629 ResultTys: {Op.getValueType(), MVT::Other},
3630 Ops: {Val.getValue(R: 1), Val.getValue(R: 0),
3631 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
3632 }
3633 return DAG.getNode(
3634 Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(),
3635 N1: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: MVT::f32, Operand: Op.getOperand(i: 0)),
3636 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3637 }
3638
3639 // Other operations are legal.
3640 return Op;
3641}
3642
3643static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
3644 const RISCVSubtarget &Subtarget) {
3645 // RISC-V FP-to-int conversions saturate to the destination register size, but
3646 // don't produce 0 for nan. We can use a conversion instruction and fix the
3647 // nan case with a compare and a select.
3648 SDValue Src = Op.getOperand(i: 0);
3649
3650 MVT DstVT = Op.getSimpleValueType();
3651 EVT SatVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3652
3653 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
3654
3655 if (!DstVT.isVector()) {
3656 // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
3657 // the result.
3658 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3659 Src.getValueType() == MVT::bf16) {
3660 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(Op), VT: MVT::f32, Operand: Src);
3661 }
3662
3663 unsigned Opc;
3664 if (SatVT == DstVT)
3665 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
3666 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3667 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
3668 else
3669 return SDValue();
3670 // FIXME: Support other SatVTs by clamping before or after the conversion.
3671
3672 SDLoc DL(Op);
3673 SDValue FpToInt = DAG.getNode(
3674 Opcode: Opc, DL, VT: DstVT, N1: Src,
3675 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: Subtarget.getXLenVT()));
3676
3677 if (Opc == RISCVISD::FCVT_WU_RV64)
3678 FpToInt = DAG.getZeroExtendInReg(Op: FpToInt, DL, VT: MVT::i32);
3679
3680 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: DstVT);
3681 return DAG.getSelectCC(DL, LHS: Src, RHS: Src, True: ZeroInt, False: FpToInt,
3682 Cond: ISD::CondCode::SETUO);
3683 }
3684
3685 // Vectors.
3686
3687 MVT DstEltVT = DstVT.getVectorElementType();
3688 MVT SrcVT = Src.getSimpleValueType();
3689 MVT SrcEltVT = SrcVT.getVectorElementType();
3690 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3691 unsigned DstEltSize = DstEltVT.getSizeInBits();
3692
3693 // Only handle saturating to the destination type.
3694 if (SatVT != DstEltVT)
3695 return SDValue();
3696
3697 MVT DstContainerVT = DstVT;
3698 MVT SrcContainerVT = SrcVT;
3699 if (DstVT.isFixedLengthVector()) {
3700 DstContainerVT = getContainerForFixedLengthVector(VT: DstVT, Subtarget);
3701 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
3702 assert(DstContainerVT.getVectorElementCount() ==
3703 SrcContainerVT.getVectorElementCount() &&
3704 "Expected same element count");
3705 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
3706 }
3707
3708 SDLoc DL(Op);
3709
3710 auto [Mask, VL] = getDefaultVLOps(VecVT: DstVT, ContainerVT: DstContainerVT, DL, DAG, Subtarget);
3711
3712 SDValue IsNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
3713 Ops: {Src, Src, DAG.getCondCode(Cond: ISD::SETNE),
3714 DAG.getUNDEF(VT: Mask.getValueType()), Mask, VL});
3715
3716 // Need to widen by more than 1 step, promote the FP type, then do a widening
3717 // convert.
3718 if (DstEltSize > (2 * SrcEltSize)) {
3719 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3720 MVT InterVT = SrcContainerVT.changeVectorElementType(EltVT: MVT::f32);
3721 Src = DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: InterVT, N1: Src, N2: Mask, N3: VL);
3722 }
3723
3724 MVT CvtContainerVT = DstContainerVT;
3725 MVT CvtEltVT = DstEltVT;
3726 if (SrcEltSize > (2 * DstEltSize)) {
3727 CvtEltVT = MVT::getIntegerVT(BitWidth: SrcEltVT.getSizeInBits() / 2);
3728 CvtContainerVT = CvtContainerVT.changeVectorElementType(EltVT: CvtEltVT);
3729 }
3730
3731 unsigned RVVOpc =
3732 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
3733 SDValue Res = DAG.getNode(Opcode: RVVOpc, DL, VT: CvtContainerVT, N1: Src, N2: Mask, N3: VL);
3734
3735 while (CvtContainerVT != DstContainerVT) {
3736 CvtEltVT = MVT::getIntegerVT(BitWidth: CvtEltVT.getSizeInBits() / 2);
3737 CvtContainerVT = CvtContainerVT.changeVectorElementType(EltVT: CvtEltVT);
3738 // Rounding mode here is arbitrary since we aren't shifting out any bits.
3739 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
3740 : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
3741 Res = DAG.getNode(Opcode: ClipOpc, DL, VT: CvtContainerVT, N1: Res, N2: Mask, N3: VL);
3742 }
3743
3744 SDValue SplatZero = DAG.getNode(
3745 Opcode: RISCVISD::VMV_V_X_VL, DL, VT: DstContainerVT, N1: DAG.getUNDEF(VT: DstContainerVT),
3746 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()), N3: VL);
3747 Res = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: DstContainerVT, N1: IsNan, N2: SplatZero,
3748 N3: Res, N4: DAG.getUNDEF(VT: DstContainerVT), N5: VL);
3749
3750 if (DstVT.isFixedLengthVector())
3751 Res = convertFromScalableVector(VT: DstVT, V: Res, DAG, Subtarget);
3752
3753 return Res;
3754}
3755
3756static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
3757 const RISCVSubtarget &Subtarget) {
3758 bool IsStrict = Op->isStrictFPOpcode();
3759 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
3760
3761 // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
3762 // bf16 conversions are always promoted to f32.
3763 if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3764 SrcVal.getValueType() == MVT::bf16) {
3765 SDLoc DL(Op);
3766 if (IsStrict) {
3767 SDValue Ext =
3768 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
3769 Ops: {Op.getOperand(i: 0), SrcVal});
3770 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
3771 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
3772 }
3773 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
3774 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: SrcVal));
3775 }
3776
3777 // Other operations are legal.
3778 return Op;
3779}
3780
3781static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
3782 switch (Opc) {
3783 case ISD::FROUNDEVEN:
3784 case ISD::STRICT_FROUNDEVEN:
3785 return RISCVFPRndMode::RNE;
3786 case ISD::FTRUNC:
3787 case ISD::STRICT_FTRUNC:
3788 return RISCVFPRndMode::RTZ;
3789 case ISD::FFLOOR:
3790 case ISD::STRICT_FFLOOR:
3791 return RISCVFPRndMode::RDN;
3792 case ISD::FCEIL:
3793 case ISD::STRICT_FCEIL:
3794 return RISCVFPRndMode::RUP;
3795 case ISD::FROUND:
3796 case ISD::LROUND:
3797 case ISD::LLROUND:
3798 case ISD::STRICT_FROUND:
3799 case ISD::STRICT_LROUND:
3800 case ISD::STRICT_LLROUND:
3801 return RISCVFPRndMode::RMM;
3802 case ISD::FRINT:
3803 case ISD::LRINT:
3804 case ISD::LLRINT:
3805 case ISD::STRICT_FRINT:
3806 case ISD::STRICT_LRINT:
3807 case ISD::STRICT_LLRINT:
3808 return RISCVFPRndMode::DYN;
3809 }
3810
3811 return RISCVFPRndMode::Invalid;
3812}
3813
3814// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
3815// the integer domain and back. Taking care to avoid converting values that are
3816// nan or already correct.
3817static SDValue
3818lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3819 const RISCVSubtarget &Subtarget) {
3820 MVT VT = Op.getSimpleValueType();
3821 assert(VT.isVector() && "Unexpected type");
3822
3823 SDLoc DL(Op);
3824
3825 SDValue Src = Op.getOperand(i: 0);
3826
3827 // Freeze the source since we are increasing the number of uses.
3828 Src = DAG.getFreeze(V: Src);
3829
3830 MVT ContainerVT = VT;
3831 if (VT.isFixedLengthVector()) {
3832 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3833 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
3834 }
3835
3836 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
3837
3838 // We do the conversion on the absolute value and fix the sign at the end.
3839 SDValue Abs = DAG.getNode(Opcode: RISCVISD::FABS_VL, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
3840
3841 // Determine the largest integer that can be represented exactly. This and
3842 // values larger than it don't have any fractional bits so don't need to
3843 // be converted.
3844 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3845 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3846 APFloat MaxVal = APFloat(FltSem);
3847 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3848 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3849 SDValue MaxValNode =
3850 DAG.getConstantFP(Val: MaxVal, DL, VT: ContainerVT.getVectorElementType());
3851 SDValue MaxValSplat = DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: ContainerVT,
3852 N1: DAG.getUNDEF(VT: ContainerVT), N2: MaxValNode, N3: VL);
3853
3854 // If abs(Src) was larger than MaxVal or nan, keep it.
3855 MVT SetccVT = MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
3856 Mask =
3857 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: SetccVT,
3858 Ops: {Abs, MaxValSplat, DAG.getCondCode(Cond: ISD::SETOLT),
3859 Mask, Mask, VL});
3860
3861 // Truncate to integer and convert back to FP.
3862 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3863 MVT XLenVT = Subtarget.getXLenVT();
3864 SDValue Truncated;
3865
3866 switch (Op.getOpcode()) {
3867 default:
3868 llvm_unreachable("Unexpected opcode");
3869 case ISD::FRINT:
3870 case ISD::FCEIL:
3871 case ISD::FFLOOR:
3872 case ISD::FROUND:
3873 case ISD::FROUNDEVEN: {
3874 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3875 assert(FRM != RISCVFPRndMode::Invalid);
3876 Truncated = DAG.getNode(Opcode: RISCVISD::VFCVT_RM_X_F_VL, DL, VT: IntVT, N1: Src, N2: Mask,
3877 N3: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), N4: VL);
3878 break;
3879 }
3880 case ISD::FTRUNC:
3881 Truncated = DAG.getNode(Opcode: RISCVISD::VFCVT_RTZ_X_F_VL, DL, VT: IntVT, N1: Src,
3882 N2: Mask, N3: VL);
3883 break;
3884 case ISD::FNEARBYINT:
3885 Truncated = DAG.getNode(Opcode: RISCVISD::VFROUND_NOEXCEPT_VL, DL, VT: ContainerVT, N1: Src,
3886 N2: Mask, N3: VL);
3887 break;
3888 }
3889
3890 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3891 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3892 Truncated = DAG.getNode(Opcode: RISCVISD::SINT_TO_FP_VL, DL, VT: ContainerVT, N1: Truncated,
3893 N2: Mask, N3: VL);
3894
3895 // Restore the original sign so that -0.0 is preserved.
3896 Truncated = DAG.getNode(Opcode: RISCVISD::FCOPYSIGN_VL, DL, VT: ContainerVT, N1: Truncated,
3897 N2: Src, N3: Src, N4: Mask, N5: VL);
3898
3899 if (!VT.isFixedLengthVector())
3900 return Truncated;
3901
3902 return convertFromScalableVector(VT, V: Truncated, DAG, Subtarget);
3903}
3904
3905// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3906// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3907// qNan and converting the new source to integer and back to FP.
3908static SDValue
3909lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3910 const RISCVSubtarget &Subtarget) {
3911 SDLoc DL(Op);
3912 MVT VT = Op.getSimpleValueType();
3913 SDValue Chain = Op.getOperand(i: 0);
3914 SDValue Src = Op.getOperand(i: 1);
3915
3916 MVT ContainerVT = VT;
3917 if (VT.isFixedLengthVector()) {
3918 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3919 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
3920 }
3921
3922 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
3923
3924 // Freeze the source since we are increasing the number of uses.
3925 Src = DAG.getFreeze(V: Src);
3926
3927 // Convert sNan to qNan by executing x + x for all unordered element x in Src.
3928 MVT MaskVT = Mask.getSimpleValueType();
3929 SDValue Unorder = DAG.getNode(Opcode: RISCVISD::STRICT_FSETCC_VL, DL,
3930 VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
3931 Ops: {Chain, Src, Src, DAG.getCondCode(Cond: ISD::SETUNE),
3932 DAG.getUNDEF(VT: MaskVT), Mask, VL});
3933 Chain = Unorder.getValue(R: 1);
3934 Src = DAG.getNode(Opcode: RISCVISD::STRICT_FADD_VL, DL,
3935 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
3936 Ops: {Chain, Src, Src, Src, Unorder, VL});
3937 Chain = Src.getValue(R: 1);
3938
3939 // We do the conversion on the absolute value and fix the sign at the end.
3940 SDValue Abs = DAG.getNode(Opcode: RISCVISD::FABS_VL, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
3941
3942 // Determine the largest integer that can be represented exactly. This and
3943 // values larger than it don't have any fractional bits so don't need to
3944 // be converted.
3945 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3946 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3947 APFloat MaxVal = APFloat(FltSem);
3948 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3949 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3950 SDValue MaxValNode =
3951 DAG.getConstantFP(Val: MaxVal, DL, VT: ContainerVT.getVectorElementType());
3952 SDValue MaxValSplat = DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: ContainerVT,
3953 N1: DAG.getUNDEF(VT: ContainerVT), N2: MaxValNode, N3: VL);
3954
3955 // If abs(Src) was larger than MaxVal or nan, keep it.
3956 Mask = DAG.getNode(
3957 Opcode: RISCVISD::SETCC_VL, DL, VT: MaskVT,
3958 Ops: {Abs, MaxValSplat, DAG.getCondCode(Cond: ISD::SETOLT), Mask, Mask, VL});
3959
3960 // Truncate to integer and convert back to FP.
3961 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3962 MVT XLenVT = Subtarget.getXLenVT();
3963 SDValue Truncated;
3964
3965 switch (Op.getOpcode()) {
3966 default:
3967 llvm_unreachable("Unexpected opcode");
3968 case ISD::STRICT_FCEIL:
3969 case ISD::STRICT_FFLOOR:
3970 case ISD::STRICT_FROUND:
3971 case ISD::STRICT_FROUNDEVEN: {
3972 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3973 assert(FRM != RISCVFPRndMode::Invalid);
3974 Truncated = DAG.getNode(
3975 Opcode: RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, VTList: DAG.getVTList(VT1: IntVT, VT2: MVT::Other),
3976 Ops: {Chain, Src, Mask, DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), VL});
3977 break;
3978 }
3979 case ISD::STRICT_FTRUNC:
3980 Truncated =
3981 DAG.getNode(Opcode: RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3982 VTList: DAG.getVTList(VT1: IntVT, VT2: MVT::Other), N1: Chain, N2: Src, N3: Mask, N4: VL);
3983 break;
3984 case ISD::STRICT_FNEARBYINT:
3985 Truncated = DAG.getNode(Opcode: RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3986 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), N1: Chain, N2: Src,
3987 N3: Mask, N4: VL);
3988 break;
3989 }
3990 Chain = Truncated.getValue(R: 1);
3991
3992 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3993 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3994 Truncated = DAG.getNode(Opcode: RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3995 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), N1: Chain,
3996 N2: Truncated, N3: Mask, N4: VL);
3997 Chain = Truncated.getValue(R: 1);
3998 }
3999
4000 // Restore the original sign so that -0.0 is preserved.
4001 Truncated = DAG.getNode(Opcode: RISCVISD::FCOPYSIGN_VL, DL, VT: ContainerVT, N1: Truncated,
4002 N2: Src, N3: Src, N4: Mask, N5: VL);
4003
4004 if (VT.isFixedLengthVector())
4005 Truncated = convertFromScalableVector(VT, V: Truncated, DAG, Subtarget);
4006 return DAG.getMergeValues(Ops: {Truncated, Chain}, dl: DL);
4007}
4008
4009static SDValue
4010lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
4011 const RISCVSubtarget &Subtarget) {
4012 MVT VT = Op.getSimpleValueType();
4013 if (VT.isVector())
4014 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
4015
4016 if (DAG.shouldOptForSize())
4017 return SDValue();
4018
4019 SDLoc DL(Op);
4020 SDValue Src = Op.getOperand(i: 0);
4021
4022 // Create an integer the size of the mantissa with the MSB set. This and all
4023 // values larger than it don't have any fractional bits so don't need to be
4024 // converted.
4025 const fltSemantics &FltSem = VT.getFltSemantics();
4026 unsigned Precision = APFloat::semanticsPrecision(FltSem);
4027 APFloat MaxVal = APFloat(FltSem);
4028 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
4029 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
4030 SDValue MaxValNode = DAG.getConstantFP(Val: MaxVal, DL, VT);
4031
4032 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
4033 return DAG.getNode(Opcode: RISCVISD::FROUND, DL, VT, N1: Src, N2: MaxValNode,
4034 N3: DAG.getTargetConstant(Val: FRM, DL, VT: Subtarget.getXLenVT()));
4035}
4036
4037// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
4038static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
4039 const RISCVSubtarget &Subtarget) {
4040 SDLoc DL(Op);
4041 MVT DstVT = Op.getSimpleValueType();
4042 SDValue Src = Op.getOperand(i: 0);
4043 MVT SrcVT = Src.getSimpleValueType();
4044 assert(SrcVT.isVector() && DstVT.isVector() &&
4045 !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
4046 "Unexpected type");
4047
4048 MVT DstContainerVT = DstVT;
4049 MVT SrcContainerVT = SrcVT;
4050
4051 if (DstVT.isFixedLengthVector()) {
4052 DstContainerVT = getContainerForFixedLengthVector(VT: DstVT, Subtarget);
4053 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
4054 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
4055 }
4056
4057 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget);
4058
4059 // [b]f16 -> f32
4060 MVT SrcElemType = SrcVT.getVectorElementType();
4061 if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
4062 MVT F32VT = SrcContainerVT.changeVectorElementType(EltVT: MVT::f32);
4063 Src = DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: F32VT, N1: Src, N2: Mask, N3: VL);
4064 }
4065
4066 SDValue Res =
4067 DAG.getNode(Opcode: RISCVISD::VFCVT_RM_X_F_VL, DL, VT: DstContainerVT, N1: Src, N2: Mask,
4068 N3: DAG.getTargetConstant(Val: matchRoundingOp(Opc: Op.getOpcode()), DL,
4069 VT: Subtarget.getXLenVT()),
4070 N4: VL);
4071
4072 if (!DstVT.isFixedLengthVector())
4073 return Res;
4074
4075 return convertFromScalableVector(VT: DstVT, V: Res, DAG, Subtarget);
4076}
4077
4078static SDValue
4079getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
4080 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
4081 SDValue Offset, SDValue Mask, SDValue VL,
4082 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4083 if (Passthru.isUndef())
4084 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4085 SDValue PolicyOp = DAG.getTargetConstant(Val: Policy, DL, VT: Subtarget.getXLenVT());
4086 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4087 return DAG.getNode(Opcode: RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
4088}
4089
4090static SDValue
4091getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
4092 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
4093 SDValue VL,
4094 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4095 if (Passthru.isUndef())
4096 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4097 SDValue PolicyOp = DAG.getTargetConstant(Val: Policy, DL, VT: Subtarget.getXLenVT());
4098 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4099 return DAG.getNode(Opcode: RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
4100}
4101
4102struct VIDSequence {
4103 int64_t StepNumerator;
4104 unsigned StepDenominator;
4105 int64_t Addend;
4106};
4107
4108static std::optional<APInt> getExactInteger(const APFloat &APF,
4109 uint32_t BitWidth) {
4110 // We will use a SINT_TO_FP to materialize this constant so we should use a
4111 // signed APSInt here.
4112 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
4113 // We use an arbitrary rounding mode here. If a floating-point is an exact
4114 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
4115 // the rounding mode changes the output value, then it is not an exact
4116 // integer.
4117 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
4118 bool IsExact;
4119 // If it is out of signed integer range, it will return an invalid operation.
4120 // If it is not an exact integer, IsExact is false.
4121 if ((APF.convertToInteger(Result&: ValInt, RM: ArbitraryRM, IsExact: &IsExact) ==
4122 APFloatBase::opInvalidOp) ||
4123 !IsExact)
4124 return std::nullopt;
4125 return ValInt.extractBits(numBits: BitWidth, bitPosition: 0);
4126}
4127
4128// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
4129// to the (non-zero) step S and start value X. This can be then lowered as the
4130// RVV sequence (VID * S) + X, for example.
4131// The step S is represented as an integer numerator divided by a positive
4132// denominator. Note that the implementation currently only identifies
4133// sequences in which either the numerator is +/- 1 or the denominator is 1. It
4134// cannot detect 2/3, for example.
4135// Note that this method will also match potentially unappealing index
4136// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
4137// determine whether this is worth generating code for.
4138//
4139// EltSizeInBits is the size of the type that the sequence will be calculated
4140// in, i.e. SEW for build_vectors or XLEN for address calculations.
4141static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
4142 unsigned EltSizeInBits) {
4143 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
4144 if (!cast<BuildVectorSDNode>(Val&: Op)->isConstant())
4145 return std::nullopt;
4146 bool IsInteger = Op.getValueType().isInteger();
4147
4148 std::optional<unsigned> SeqStepDenom;
4149 std::optional<APInt> SeqStepNum;
4150 std::optional<APInt> SeqAddend;
4151 std::optional<std::pair<APInt, unsigned>> PrevElt;
4152 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
4153
4154 // First extract the ops into a list of constant integer values. This may not
4155 // be possible for floats if they're not all representable as integers.
4156 SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
4157 const unsigned OpSize = Op.getScalarValueSizeInBits();
4158 for (auto [Idx, Elt] : enumerate(First: Op->op_values())) {
4159 if (Elt.isUndef()) {
4160 Elts[Idx] = std::nullopt;
4161 continue;
4162 }
4163 if (IsInteger) {
4164 Elts[Idx] = Elt->getAsAPIntVal().trunc(width: OpSize).zext(width: EltSizeInBits);
4165 } else {
4166 auto ExactInteger =
4167 getExactInteger(APF: cast<ConstantFPSDNode>(Val: Elt)->getValueAPF(), BitWidth: OpSize);
4168 if (!ExactInteger)
4169 return std::nullopt;
4170 Elts[Idx] = *ExactInteger;
4171 }
4172 }
4173
4174 for (auto [Idx, Elt] : enumerate(First&: Elts)) {
4175 // Assume undef elements match the sequence; we just have to be careful
4176 // when interpolating across them.
4177 if (!Elt)
4178 continue;
4179
4180 if (PrevElt) {
4181 // Calculate the step since the last non-undef element, and ensure
4182 // it's consistent across the entire sequence.
4183 unsigned IdxDiff = Idx - PrevElt->second;
4184 APInt ValDiff = *Elt - PrevElt->first;
4185
4186 // A zero-value value difference means that we're somewhere in the middle
4187 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
4188 // step change before evaluating the sequence.
4189 if (ValDiff == 0)
4190 continue;
4191
4192 int64_t Remainder = ValDiff.srem(RHS: IdxDiff);
4193 // Normalize the step if it's greater than 1.
4194 if (Remainder != ValDiff.getSExtValue()) {
4195 // The difference must cleanly divide the element span.
4196 if (Remainder != 0)
4197 return std::nullopt;
4198 ValDiff = ValDiff.sdiv(RHS: IdxDiff);
4199 IdxDiff = 1;
4200 }
4201
4202 if (!SeqStepNum)
4203 SeqStepNum = ValDiff;
4204 else if (ValDiff != SeqStepNum)
4205 return std::nullopt;
4206
4207 if (!SeqStepDenom)
4208 SeqStepDenom = IdxDiff;
4209 else if (IdxDiff != *SeqStepDenom)
4210 return std::nullopt;
4211 }
4212
4213 // Record this non-undef element for later.
4214 if (!PrevElt || PrevElt->first != *Elt)
4215 PrevElt = std::make_pair(x&: *Elt, y&: Idx);
4216 }
4217
4218 // We need to have logged a step for this to count as a legal index sequence.
4219 if (!SeqStepNum || !SeqStepDenom)
4220 return std::nullopt;
4221
4222 // Loop back through the sequence and validate elements we might have skipped
4223 // while waiting for a valid step. While doing this, log any sequence addend.
4224 for (auto [Idx, Elt] : enumerate(First&: Elts)) {
4225 if (!Elt)
4226 continue;
4227 APInt ExpectedVal =
4228 (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
4229 *SeqStepNum)
4230 .sdiv(RHS: *SeqStepDenom);
4231
4232 APInt Addend = *Elt - ExpectedVal;
4233 if (!SeqAddend)
4234 SeqAddend = Addend;
4235 else if (Addend != SeqAddend)
4236 return std::nullopt;
4237 }
4238
4239 assert(SeqAddend && "Must have an addend if we have a step");
4240
4241 return VIDSequence{.StepNumerator: SeqStepNum->getSExtValue(), .StepDenominator: *SeqStepDenom,
4242 .Addend: SeqAddend->getSExtValue()};
4243}
4244
4245// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
4246// and lower it as a VRGATHER_VX_VL from the source vector.
4247static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
4248 SelectionDAG &DAG,
4249 const RISCVSubtarget &Subtarget) {
4250 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
4251 return SDValue();
4252 SDValue Src = SplatVal.getOperand(i: 0);
4253 // Don't perform this optimization for i1 vectors, or if the element types are
4254 // different
4255 // FIXME: Support i1 vectors, maybe by promoting to i8?
4256 MVT EltTy = VT.getVectorElementType();
4257 if (EltTy == MVT::i1 ||
4258 !DAG.getTargetLoweringInfo().isTypeLegal(VT: Src.getValueType()))
4259 return SDValue();
4260 MVT SrcVT = Src.getSimpleValueType();
4261 if (EltTy != SrcVT.getVectorElementType())
4262 return SDValue();
4263 SDValue Idx = SplatVal.getOperand(i: 1);
4264 // The index must be a legal type.
4265 if (Idx.getValueType() != Subtarget.getXLenVT())
4266 return SDValue();
4267
4268 // Check that we know Idx lies within VT
4269 if (!TypeSize::isKnownLE(LHS: SrcVT.getSizeInBits(), RHS: VT.getSizeInBits())) {
4270 auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
4271 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
4272 return SDValue();
4273 }
4274
4275 // Convert fixed length vectors to scalable
4276 MVT ContainerVT = VT;
4277 if (VT.isFixedLengthVector())
4278 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4279
4280 MVT SrcContainerVT = SrcVT;
4281 if (SrcVT.isFixedLengthVector()) {
4282 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
4283 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
4284 }
4285
4286 // Put Vec in a VT sized vector
4287 if (SrcContainerVT.getVectorMinNumElements() <
4288 ContainerVT.getVectorMinNumElements())
4289 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: Src, Idx: 0);
4290 else
4291 Src = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec: Src, Idx: 0);
4292
4293 // We checked that Idx fits inside VT earlier
4294 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4295 SDValue Gather = DAG.getNode(Opcode: RISCVISD::VRGATHER_VX_VL, DL, VT: ContainerVT, N1: Src,
4296 N2: Idx, N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
4297 if (VT.isFixedLengthVector())
4298 Gather = convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
4299 return Gather;
4300}
4301
4302static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
4303 const RISCVSubtarget &Subtarget) {
4304 MVT VT = Op.getSimpleValueType();
4305 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4306
4307 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4308
4309 SDLoc DL(Op);
4310 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4311
4312 if (auto SimpleVID = isSimpleVIDSequence(Op, EltSizeInBits: Op.getScalarValueSizeInBits())) {
4313 int64_t StepNumerator = SimpleVID->StepNumerator;
4314 unsigned StepDenominator = SimpleVID->StepDenominator;
4315 int64_t Addend = SimpleVID->Addend;
4316
4317 assert(StepNumerator != 0 && "Invalid step");
4318 bool Negate = false;
4319 int64_t SplatStepVal = StepNumerator;
4320 unsigned StepOpcode = ISD::MUL;
4321 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
4322 // anyway as the shift of 63 won't fit in uimm5.
4323 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
4324 isPowerOf2_64(Value: std::abs(i: StepNumerator))) {
4325 Negate = StepNumerator < 0;
4326 StepOpcode = ISD::SHL;
4327 SplatStepVal = Log2_64(Value: std::abs(i: StepNumerator));
4328 }
4329
4330 // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
4331 // since it's the immediate value many RVV instructions accept. There is
4332 // no vmul.vi instruction so ensure multiply constant can fit in a
4333 // single addi instruction. For the addend, we allow up to 32 bits..
4334 if (((StepOpcode == ISD::MUL && isInt<12>(x: SplatStepVal)) ||
4335 (StepOpcode == ISD::SHL && isUInt<5>(x: SplatStepVal))) &&
4336 isPowerOf2_32(Value: StepDenominator) &&
4337 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(x: Addend)) {
4338 MVT VIDVT =
4339 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
4340 MVT VIDContainerVT = getContainerForFixedLengthVector(VT: VIDVT, Subtarget);
4341 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: VIDContainerVT, N1: Mask, N2: VL);
4342 // Convert right out of the scalable type so we can use standard ISD
4343 // nodes for the rest of the computation. If we used scalable types with
4344 // these, we'd lose the fixed-length vector info and generate worse
4345 // vsetvli code.
4346 VID = convertFromScalableVector(VT: VIDVT, V: VID, DAG, Subtarget);
4347 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
4348 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
4349 SDValue SplatStep = DAG.getSignedConstant(Val: SplatStepVal, DL, VT: VIDVT);
4350 VID = DAG.getNode(Opcode: StepOpcode, DL, VT: VIDVT, N1: VID, N2: SplatStep);
4351 }
4352 if (StepDenominator != 1) {
4353 SDValue SplatStep =
4354 DAG.getConstant(Val: Log2_64(Value: StepDenominator), DL, VT: VIDVT);
4355 VID = DAG.getNode(Opcode: ISD::SRL, DL, VT: VIDVT, N1: VID, N2: SplatStep);
4356 }
4357 if (Addend != 0 || Negate) {
4358 SDValue SplatAddend = DAG.getSignedConstant(Val: Addend, DL, VT: VIDVT);
4359 VID = DAG.getNode(Opcode: Negate ? ISD::SUB : ISD::ADD, DL, VT: VIDVT, N1: SplatAddend,
4360 N2: VID);
4361 }
4362 if (VT.isFloatingPoint()) {
4363 // TODO: Use vfwcvt to reduce register pressure.
4364 VID = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: VID);
4365 }
4366 return VID;
4367 }
4368 }
4369
4370 return SDValue();
4371}
4372
4373/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
4374/// which constitute a large proportion of the elements. In such cases we can
4375/// splat a vector with the dominant element and make up the shortfall with
4376/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
4377/// Note that this includes vectors of 2 elements by association. The
4378/// upper-most element is the "dominant" one, allowing us to use a splat to
4379/// "insert" the upper element, and an insert of the lower element at position
4380/// 0, which improves codegen.
4381static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
4382 const RISCVSubtarget &Subtarget) {
4383 MVT VT = Op.getSimpleValueType();
4384 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4385
4386 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4387
4388 SDLoc DL(Op);
4389 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4390
4391 MVT XLenVT = Subtarget.getXLenVT();
4392 unsigned NumElts = Op.getNumOperands();
4393
4394 SDValue DominantValue;
4395 unsigned MostCommonCount = 0;
4396 DenseMap<SDValue, unsigned> ValueCounts;
4397 unsigned NumUndefElts =
4398 count_if(Range: Op->op_values(), P: [](const SDValue &V) { return V.isUndef(); });
4399
4400 // Track the number of scalar loads we know we'd be inserting, estimated as
4401 // any non-zero floating-point constant. Other kinds of element are either
4402 // already in registers or are materialized on demand. The threshold at which
4403 // a vector load is more desirable than several scalar materializion and
4404 // vector-insertion instructions is not known.
4405 unsigned NumScalarLoads = 0;
4406
4407 for (SDValue V : Op->op_values()) {
4408 if (V.isUndef())
4409 continue;
4410
4411 unsigned &Count = ValueCounts[V];
4412 if (0 == Count)
4413 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: V))
4414 NumScalarLoads += !CFP->isPosZero();
4415
4416 // Is this value dominant? In case of a tie, prefer the highest element as
4417 // it's cheaper to insert near the beginning of a vector than it is at the
4418 // end.
4419 if (++Count >= MostCommonCount) {
4420 DominantValue = V;
4421 MostCommonCount = Count;
4422 }
4423 }
4424
4425 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
4426 unsigned NumDefElts = NumElts - NumUndefElts;
4427 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
4428
4429 // Don't perform this optimization when optimizing for size, since
4430 // materializing elements and inserting them tends to cause code bloat.
4431 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
4432 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode())) &&
4433 ((MostCommonCount > DominantValueCountThreshold) ||
4434 (ValueCounts.size() <= Log2_32(Value: NumDefElts)))) {
4435 // Start by splatting the most common element.
4436 SDValue Vec = DAG.getSplatBuildVector(VT, DL, Op: DominantValue);
4437
4438 DenseSet<SDValue> Processed{DominantValue};
4439
4440 // We can handle an insert into the last element (of a splat) via
4441 // v(f)slide1down. This is slightly better than the vslideup insert
4442 // lowering as it avoids the need for a vector group temporary. It
4443 // is also better than using vmerge.vx as it avoids the need to
4444 // materialize the mask in a vector register.
4445 if (SDValue LastOp = Op->getOperand(Num: Op->getNumOperands() - 1);
4446 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
4447 LastOp != DominantValue) {
4448 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
4449 auto OpCode =
4450 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4451 if (!VT.isFloatingPoint())
4452 LastOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: LastOp);
4453 Vec = DAG.getNode(Opcode: OpCode, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Vec,
4454 N3: LastOp, N4: Mask, N5: VL);
4455 Vec = convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
4456 Processed.insert(V: LastOp);
4457 }
4458
4459 MVT SelMaskTy = VT.changeVectorElementType(EltVT: MVT::i1);
4460 for (const auto &OpIdx : enumerate(First: Op->ops())) {
4461 const SDValue &V = OpIdx.value();
4462 if (V.isUndef() || !Processed.insert(V).second)
4463 continue;
4464 if (ValueCounts[V] == 1) {
4465 Vec = DAG.getInsertVectorElt(DL, Vec, Elt: V, Idx: OpIdx.index());
4466 } else {
4467 // Blend in all instances of this value using a VSELECT, using a
4468 // mask where each bit signals whether that element is the one
4469 // we're after.
4470 SmallVector<SDValue> Ops;
4471 transform(Range: Op->op_values(), d_first: std::back_inserter(x&: Ops), F: [&](SDValue V1) {
4472 return DAG.getConstant(Val: V == V1, DL, VT: XLenVT);
4473 });
4474 Vec = DAG.getNode(Opcode: ISD::VSELECT, DL, VT,
4475 N1: DAG.getBuildVector(VT: SelMaskTy, DL, Ops),
4476 N2: DAG.getSplatBuildVector(VT, DL, Op: V), N3: Vec);
4477 }
4478 }
4479
4480 return Vec;
4481 }
4482
4483 return SDValue();
4484}
4485
4486static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
4487 const RISCVSubtarget &Subtarget) {
4488 MVT VT = Op.getSimpleValueType();
4489 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4490
4491 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4492
4493 SDLoc DL(Op);
4494 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4495
4496 MVT XLenVT = Subtarget.getXLenVT();
4497 unsigned NumElts = Op.getNumOperands();
4498
4499 if (VT.getVectorElementType() == MVT::i1) {
4500 if (ISD::isBuildVectorAllZeros(N: Op.getNode())) {
4501 SDValue VMClr = DAG.getNode(Opcode: RISCVISD::VMCLR_VL, DL, VT: ContainerVT, Operand: VL);
4502 return convertFromScalableVector(VT, V: VMClr, DAG, Subtarget);
4503 }
4504
4505 if (ISD::isBuildVectorAllOnes(N: Op.getNode())) {
4506 SDValue VMSet = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
4507 return convertFromScalableVector(VT, V: VMSet, DAG, Subtarget);
4508 }
4509
4510 // Lower constant mask BUILD_VECTORs via an integer vector type, in
4511 // scalar integer chunks whose bit-width depends on the number of mask
4512 // bits and XLEN.
4513 // First, determine the most appropriate scalar integer type to use. This
4514 // is at most XLenVT, but may be shrunk to a smaller vector element type
4515 // according to the size of the final vector - use i8 chunks rather than
4516 // XLenVT if we're producing a v8i1. This results in more consistent
4517 // codegen across RV32 and RV64.
4518 unsigned NumViaIntegerBits = std::clamp(val: NumElts, lo: 8u, hi: Subtarget.getXLen());
4519 NumViaIntegerBits = std::min(a: NumViaIntegerBits, b: Subtarget.getELen());
4520 // If we have to use more than one INSERT_VECTOR_ELT then this
4521 // optimization is likely to increase code size; avoid performing it in
4522 // such a case. We can use a load from a constant pool in this case.
4523 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
4524 return SDValue();
4525 // Now we can create our integer vector type. Note that it may be larger
4526 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
4527 unsigned IntegerViaVecElts = divideCeil(Numerator: NumElts, Denominator: NumViaIntegerBits);
4528 MVT IntegerViaVecVT =
4529 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NumViaIntegerBits),
4530 NumElements: IntegerViaVecElts);
4531
4532 uint64_t Bits = 0;
4533 unsigned BitPos = 0, IntegerEltIdx = 0;
4534 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
4535
4536 for (unsigned I = 0; I < NumElts;) {
4537 SDValue V = Op.getOperand(i: I);
4538 bool BitValue = !V.isUndef() && V->getAsZExtVal();
4539 Bits |= ((uint64_t)BitValue << BitPos);
4540 ++BitPos;
4541 ++I;
4542
4543 // Once we accumulate enough bits to fill our scalar type or process the
4544 // last element, insert into our vector and clear our accumulated data.
4545 if (I % NumViaIntegerBits == 0 || I == NumElts) {
4546 if (NumViaIntegerBits <= 32)
4547 Bits = SignExtend64<32>(x: Bits);
4548 SDValue Elt = DAG.getSignedConstant(Val: Bits, DL, VT: XLenVT);
4549 Elts[IntegerEltIdx] = Elt;
4550 Bits = 0;
4551 BitPos = 0;
4552 IntegerEltIdx++;
4553 }
4554 }
4555
4556 SDValue Vec = DAG.getBuildVector(VT: IntegerViaVecVT, DL, Ops: Elts);
4557
4558 if (NumElts < NumViaIntegerBits) {
4559 // If we're producing a smaller vector than our minimum legal integer
4560 // type, bitcast to the equivalent (known-legal) mask type, and extract
4561 // our final mask.
4562 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
4563 Vec = DAG.getBitcast(VT: MVT::v8i1, V: Vec);
4564 Vec = DAG.getExtractSubvector(DL, VT, Vec, Idx: 0);
4565 } else {
4566 // Else we must have produced an integer type with the same size as the
4567 // mask type; bitcast for the final result.
4568 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
4569 Vec = DAG.getBitcast(VT, V: Vec);
4570 }
4571
4572 return Vec;
4573 }
4574
4575 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4576 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4577 : RISCVISD::VMV_V_X_VL;
4578 if (!VT.isFloatingPoint())
4579 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Splat);
4580 Splat =
4581 DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Splat, N3: VL);
4582 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
4583 }
4584
4585 // Try and match index sequences, which we can lower to the vid instruction
4586 // with optional modifications. An all-undef vector is matched by
4587 // getSplatValue, above.
4588 if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
4589 return Res;
4590
4591 // For very small build_vectors, use a single scalar insert of a constant.
4592 // TODO: Base this on constant rematerialization cost, not size.
4593 const unsigned EltBitSize = VT.getScalarSizeInBits();
4594 if (VT.getSizeInBits() <= 32 &&
4595 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode())) {
4596 MVT ViaIntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
4597 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
4598 "Unexpected sequence type");
4599 // If we can use the original VL with the modified element type, this
4600 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4601 // be moved into InsertVSETVLI?
4602 unsigned ViaVecLen =
4603 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
4604 MVT ViaVecVT = MVT::getVectorVT(VT: ViaIntVT, NumElements: ViaVecLen);
4605
4606 uint64_t EltMask = maskTrailingOnes<uint64_t>(N: EltBitSize);
4607 uint64_t SplatValue = 0;
4608 // Construct the amalgamated value at this larger vector type.
4609 for (const auto &OpIdx : enumerate(First: Op->op_values())) {
4610 const auto &SeqV = OpIdx.value();
4611 if (!SeqV.isUndef())
4612 SplatValue |=
4613 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
4614 }
4615
4616 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4617 // achieve better constant materializion.
4618 // On RV32, we need to sign-extend to use getSignedConstant.
4619 if (ViaIntVT == MVT::i32)
4620 SplatValue = SignExtend64<32>(x: SplatValue);
4621
4622 SDValue Vec = DAG.getInsertVectorElt(
4623 DL, Vec: DAG.getUNDEF(VT: ViaVecVT),
4624 Elt: DAG.getSignedConstant(Val: SplatValue, DL, VT: XLenVT), Idx: 0);
4625 if (ViaVecLen != 1)
4626 Vec = DAG.getExtractSubvector(DL, VT: MVT::getVectorVT(VT: ViaIntVT, NumElements: 1), Vec, Idx: 0);
4627 return DAG.getBitcast(VT, V: Vec);
4628 }
4629
4630
4631 // Attempt to detect "hidden" splats, which only reveal themselves as splats
4632 // when re-interpreted as a vector with a larger element type. For example,
4633 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
4634 // could be instead splat as
4635 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
4636 // TODO: This optimization could also work on non-constant splats, but it
4637 // would require bit-manipulation instructions to construct the splat value.
4638 SmallVector<SDValue> Sequence;
4639 const auto *BV = cast<BuildVectorSDNode>(Val&: Op);
4640 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
4641 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode()) &&
4642 BV->getRepeatedSequence(Sequence) &&
4643 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
4644 unsigned SeqLen = Sequence.size();
4645 MVT ViaIntVT = MVT::getIntegerVT(BitWidth: EltBitSize * SeqLen);
4646 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
4647 ViaIntVT == MVT::i64) &&
4648 "Unexpected sequence type");
4649
4650 // If we can use the original VL with the modified element type, this
4651 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4652 // be moved into InsertVSETVLI?
4653 const unsigned RequiredVL = NumElts / SeqLen;
4654 const unsigned ViaVecLen =
4655 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
4656 NumElts : RequiredVL;
4657 MVT ViaVecVT = MVT::getVectorVT(VT: ViaIntVT, NumElements: ViaVecLen);
4658
4659 unsigned EltIdx = 0;
4660 uint64_t EltMask = maskTrailingOnes<uint64_t>(N: EltBitSize);
4661 uint64_t SplatValue = 0;
4662 // Construct the amalgamated value which can be splatted as this larger
4663 // vector type.
4664 for (const auto &SeqV : Sequence) {
4665 if (!SeqV.isUndef())
4666 SplatValue |=
4667 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4668 EltIdx++;
4669 }
4670
4671 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4672 // achieve better constant materializion.
4673 // On RV32, we need to sign-extend to use getSignedConstant.
4674 if (ViaIntVT == MVT::i32)
4675 SplatValue = SignExtend64<32>(x: SplatValue);
4676
4677 // Since we can't introduce illegal i64 types at this stage, we can only
4678 // perform an i64 splat on RV32 if it is its own sign-extended value. That
4679 // way we can use RVV instructions to splat.
4680 assert((ViaIntVT.bitsLE(XLenVT) ||
4681 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
4682 "Unexpected bitcast sequence");
4683 if (ViaIntVT.bitsLE(VT: XLenVT) || isInt<32>(x: SplatValue)) {
4684 SDValue ViaVL =
4685 DAG.getConstant(Val: ViaVecVT.getVectorNumElements(), DL, VT: XLenVT);
4686 MVT ViaContainerVT =
4687 getContainerForFixedLengthVector(VT: ViaVecVT, Subtarget);
4688 SDValue Splat =
4689 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ViaContainerVT,
4690 N1: DAG.getUNDEF(VT: ViaContainerVT),
4691 N2: DAG.getSignedConstant(Val: SplatValue, DL, VT: XLenVT), N3: ViaVL);
4692 Splat = convertFromScalableVector(VT: ViaVecVT, V: Splat, DAG, Subtarget);
4693 if (ViaVecLen != RequiredVL)
4694 Splat = DAG.getExtractSubvector(
4695 DL, VT: MVT::getVectorVT(VT: ViaIntVT, NumElements: RequiredVL), Vec: Splat, Idx: 0);
4696 return DAG.getBitcast(VT, V: Splat);
4697 }
4698 }
4699
4700 // If the number of signbits allows, see if we can lower as a <N x i8>.
4701 // Our main goal here is to reduce LMUL (and thus work) required to
4702 // build the constant, but we will also narrow if the resulting
4703 // narrow vector is known to materialize cheaply.
4704 // TODO: We really should be costing the smaller vector. There are
4705 // profitable cases this misses.
4706 if (EltBitSize > 8 && VT.isInteger() &&
4707 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
4708 DAG.ComputeMaxSignificantBits(Op) <= 8) {
4709 SDValue Source = DAG.getBuildVector(VT: VT.changeVectorElementType(EltVT: MVT::i8),
4710 DL, Ops: Op->ops());
4711 Source = convertToScalableVector(VT: ContainerVT.changeVectorElementType(EltVT: MVT::i8),
4712 V: Source, DAG, Subtarget);
4713 SDValue Res = DAG.getNode(Opcode: RISCVISD::VSEXT_VL, DL, VT: ContainerVT, N1: Source, N2: Mask, N3: VL);
4714 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
4715 }
4716
4717 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4718 return Res;
4719
4720 // For constant vectors, use generic constant pool lowering. Otherwise,
4721 // we'd have to materialize constants in GPRs just to move them into the
4722 // vector.
4723 return SDValue();
4724}
4725
4726static unsigned getPACKOpcode(unsigned DestBW,
4727 const RISCVSubtarget &Subtarget) {
4728 switch (DestBW) {
4729 default:
4730 llvm_unreachable("Unsupported pack size");
4731 case 16:
4732 return RISCV::PACKH;
4733 case 32:
4734 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
4735 case 64:
4736 assert(Subtarget.is64Bit());
4737 return RISCV::PACK;
4738 }
4739}
4740
4741/// Double the element size of the build vector to reduce the number
4742/// of vslide1down in the build vector chain. In the worst case, this
4743/// trades three scalar operations for 1 vector operation. Scalar
4744/// operations are generally lower latency, and for out-of-order cores
4745/// we also benefit from additional parallelism.
4746static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
4747 const RISCVSubtarget &Subtarget) {
4748 SDLoc DL(Op);
4749 MVT VT = Op.getSimpleValueType();
4750 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4751 MVT ElemVT = VT.getVectorElementType();
4752 if (!ElemVT.isInteger())
4753 return SDValue();
4754
4755 // TODO: Relax these architectural restrictions, possibly with costing
4756 // of the actual instructions required.
4757 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4758 return SDValue();
4759
4760 unsigned NumElts = VT.getVectorNumElements();
4761 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
4762 if (ElemSizeInBits >= std::min(a: Subtarget.getELen(), b: Subtarget.getXLen()) ||
4763 NumElts % 2 != 0)
4764 return SDValue();
4765
4766 // Produce [B,A] packed into a type twice as wide. Note that all
4767 // scalars are XLenVT, possibly masked (see below).
4768 MVT XLenVT = Subtarget.getXLenVT();
4769 SDValue Mask = DAG.getConstant(
4770 Val: APInt::getLowBitsSet(numBits: XLenVT.getSizeInBits(), loBitsSet: ElemSizeInBits), DL, VT: XLenVT);
4771 auto pack = [&](SDValue A, SDValue B) {
4772 // Bias the scheduling of the inserted operations to near the
4773 // definition of the element - this tends to reduce register
4774 // pressure overall.
4775 SDLoc ElemDL(B);
4776 if (Subtarget.hasStdExtZbkb())
4777 // Note that we're relying on the high bits of the result being
4778 // don't care. For PACKW, the result is *sign* extended.
4779 return SDValue(
4780 DAG.getMachineNode(Opcode: getPACKOpcode(DestBW: ElemSizeInBits * 2, Subtarget),
4781 dl: ElemDL, VT: XLenVT, Op1: A, Op2: B),
4782 0);
4783
4784 A = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(A), VT: XLenVT, N1: A, N2: Mask);
4785 B = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(B), VT: XLenVT, N1: B, N2: Mask);
4786 SDValue ShtAmt = DAG.getConstant(Val: ElemSizeInBits, DL: ElemDL, VT: XLenVT);
4787 return DAG.getNode(Opcode: ISD::OR, DL: ElemDL, VT: XLenVT, N1: A,
4788 N2: DAG.getNode(Opcode: ISD::SHL, DL: ElemDL, VT: XLenVT, N1: B, N2: ShtAmt),
4789 Flags: SDNodeFlags::Disjoint);
4790 };
4791
4792 SmallVector<SDValue> NewOperands;
4793 NewOperands.reserve(N: NumElts / 2);
4794 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4795 NewOperands.push_back(Elt: pack(Op.getOperand(i), Op.getOperand(i: i + 1)));
4796 assert(NumElts == NewOperands.size() * 2);
4797 MVT WideVT = MVT::getIntegerVT(BitWidth: ElemSizeInBits * 2);
4798 MVT WideVecVT = MVT::getVectorVT(VT: WideVT, NumElements: NumElts / 2);
4799 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
4800 Operand: DAG.getBuildVector(VT: WideVecVT, DL, Ops: NewOperands));
4801}
4802
4803static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4804 const RISCVSubtarget &Subtarget) {
4805 MVT VT = Op.getSimpleValueType();
4806 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4807
4808 MVT EltVT = VT.getVectorElementType();
4809 MVT XLenVT = Subtarget.getXLenVT();
4810
4811 SDLoc DL(Op);
4812
4813 if (!Subtarget.is64Bit() && Subtarget.hasStdExtP()) {
4814 if (VT == MVT::v2i16) {
4815 SDValue Lo = DAG.getBitcast(
4816 VT: MVT::v2i16,
4817 V: DAG.getAnyExtOrTrunc(Op: Op->getOperand(Num: 0), DL, VT: MVT::i32));
4818 SDValue Hi = DAG.getBitcast(
4819 VT: MVT::v2i16,
4820 V: DAG.getAnyExtOrTrunc(Op: Op->getOperand(Num: 1), DL, VT: MVT::i32));
4821 return DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: MVT::v2i16, N1: Lo, N2: Hi);
4822 }
4823
4824 if (VT == MVT::v4i8) {
4825 // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b,
4826 // d))
4827 SDValue Val0 =
4828 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 0));
4829 SDValue Val1 =
4830 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 1));
4831 SDValue Val2 =
4832 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 2));
4833 SDValue Val3 =
4834 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 3));
4835 SDValue Concat1 =
4836 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i8, N1: Val0, N2: Val2);
4837 SDValue Concat2 =
4838 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i8, N1: Val1, N2: Val3);
4839 SDValue PPairE =
4840 DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: MVT::v8i8, N1: Concat1, N2: Concat2);
4841
4842 SDValue Lo = DAG.getExtractSubvector(DL, VT: MVT::v4i8, Vec: PPairE, Idx: 0);
4843 SDValue Hi = DAG.getExtractSubvector(DL, VT: MVT::v4i8, Vec: PPairE, Idx: 4);
4844
4845 return DAG.getBitcast(VT: MVT::v4i8,
4846 V: DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: MVT::v2i16,
4847 N1: DAG.getBitcast(VT: MVT::v2i16, V: Lo),
4848 N2: DAG.getBitcast(VT: MVT::v2i16, V: Hi)));
4849 }
4850
4851 llvm_unreachable("Unexpected RV32 P BUILD_VECTOR type");
4852 }
4853
4854 // Proper support for f16 requires Zvfh. bf16 always requires special
4855 // handling. We need to cast the scalar to integer and create an integer
4856 // build_vector.
4857 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
4858 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
4859 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
4860 SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
4861 for (const auto &[I, U] : enumerate(First: Op->ops())) {
4862 SDValue Elem = U.get();
4863 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4864 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4865 // Called by LegalizeDAG, we need to use XLenVT operations since we
4866 // can't create illegal types.
4867 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: Elem)) {
4868 // Manually constant fold so the integer build_vector can be lowered
4869 // better. Waiting for DAGCombine will be too late.
4870 APInt V =
4871 C->getValueAPF().bitcastToAPInt().sext(width: XLenVT.getSizeInBits());
4872 NewOps[I] = DAG.getConstant(Val: V, DL, VT: XLenVT);
4873 } else {
4874 NewOps[I] = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Elem);
4875 }
4876 } else {
4877 // Called by scalar type legalizer, we can use i16.
4878 NewOps[I] = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: I));
4879 }
4880 }
4881 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: IVT, Ops: NewOps);
4882 return DAG.getBitcast(VT, V: Res);
4883 }
4884
4885 if (ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode()) ||
4886 ISD::isBuildVectorOfConstantFPSDNodes(N: Op.getNode()))
4887 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4888
4889 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4890
4891 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4892
4893 if (VT.getVectorElementType() == MVT::i1) {
4894 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4895 // vector type, we have a legal equivalently-sized i8 type, so we can use
4896 // that.
4897 MVT WideVecVT = VT.changeVectorElementType(EltVT: MVT::i8);
4898 SDValue VecZero = DAG.getConstant(Val: 0, DL, VT: WideVecVT);
4899
4900 SDValue WideVec;
4901 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4902 // For a splat, perform a scalar truncate before creating the wider
4903 // vector.
4904 Splat = DAG.getNode(Opcode: ISD::AND, DL, VT: Splat.getValueType(), N1: Splat,
4905 N2: DAG.getConstant(Val: 1, DL, VT: Splat.getValueType()));
4906 WideVec = DAG.getSplatBuildVector(VT: WideVecVT, DL, Op: Splat);
4907 } else {
4908 SmallVector<SDValue, 8> Ops(Op->op_values());
4909 WideVec = DAG.getBuildVector(VT: WideVecVT, DL, Ops);
4910 SDValue VecOne = DAG.getConstant(Val: 1, DL, VT: WideVecVT);
4911 WideVec = DAG.getNode(Opcode: ISD::AND, DL, VT: WideVecVT, N1: WideVec, N2: VecOne);
4912 }
4913
4914 return DAG.getSetCC(DL, VT, LHS: WideVec, RHS: VecZero, Cond: ISD::SETNE);
4915 }
4916
4917 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4918 if (auto Gather = matchSplatAsGather(SplatVal: Splat, VT, DL, DAG, Subtarget))
4919 return Gather;
4920
4921 if (!VT.isFloatingPoint())
4922 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Splat);
4923
4924 // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
4925 // pressure at high LMUL.
4926 bool IsScalar = all_of(Range: Op->ops().drop_front(),
4927 P: [](const SDUse &U) { return U.get().isUndef(); });
4928 unsigned Opc =
4929 VT.isFloatingPoint()
4930 ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
4931 : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
4932 Splat =
4933 DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Splat, N3: VL);
4934 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
4935 }
4936
4937 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4938 return Res;
4939
4940 // If we're compiling for an exact VLEN value, we can split our work per
4941 // register in the register group.
4942 if (const auto VLen = Subtarget.getRealVLen();
4943 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4944 MVT ElemVT = VT.getVectorElementType();
4945 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
4946 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4947 MVT OneRegVT = MVT::getVectorVT(VT: ElemVT, NumElements: ElemsPerVReg);
4948 MVT M1VT = getContainerForFixedLengthVector(VT: OneRegVT, Subtarget);
4949 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
4950
4951 // The following semantically builds up a fixed length concat_vector
4952 // of the component build_vectors. We eagerly lower to scalable and
4953 // insert_subvector here to avoid DAG combining it back to a large
4954 // build_vector.
4955 SmallVector<SDValue> BuildVectorOps(Op->ops());
4956 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4957 SDValue Vec = DAG.getUNDEF(VT: ContainerVT);
4958 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4959 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(N: i, M: ElemsPerVReg);
4960 SDValue SubBV =
4961 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: OneRegVT, Ops: OneVRegOfOps);
4962 SubBV = convertToScalableVector(VT: M1VT, V: SubBV, DAG, Subtarget);
4963 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4964 Vec = DAG.getInsertSubvector(DL, Vec, SubVec: SubBV, Idx: InsertIdx);
4965 }
4966 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
4967 }
4968
4969 // If we're about to resort to vslide1down (or stack usage), pack our
4970 // elements into the widest scalar type we can. This will force a VL/VTYPE
4971 // toggle, but reduces the critical path, the number of vslide1down ops
4972 // required, and possibly enables scalar folds of the values.
4973 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4974 return Res;
4975
4976 // For m1 vectors, if we have non-undef values in both halves of our vector,
4977 // split the vector into low and high halves, build them separately, then
4978 // use a vselect to combine them. For long vectors, this cuts the critical
4979 // path of the vslide1down sequence in half, and gives us an opportunity
4980 // to special case each half independently. Note that we don't change the
4981 // length of the sub-vectors here, so if both fallback to the generic
4982 // vslide1down path, we should be able to fold the vselect into the final
4983 // vslidedown (for the undef tail) for the first half w/ masking.
4984 unsigned NumElts = VT.getVectorNumElements();
4985 unsigned NumUndefElts =
4986 count_if(Range: Op->op_values(), P: [](const SDValue &V) { return V.isUndef(); });
4987 unsigned NumDefElts = NumElts - NumUndefElts;
4988 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4989 ContainerVT.bitsLE(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT))) {
4990 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4991 SmallVector<SDValue> MaskVals;
4992 SDValue UndefElem = DAG.getUNDEF(VT: Op->getOperand(Num: 0)->getValueType(ResNo: 0));
4993 SubVecAOps.reserve(N: NumElts);
4994 SubVecBOps.reserve(N: NumElts);
4995 for (const auto &[Idx, U] : enumerate(First: Op->ops())) {
4996 SDValue Elem = U.get();
4997 if (Idx < NumElts / 2) {
4998 SubVecAOps.push_back(Elt: Elem);
4999 SubVecBOps.push_back(Elt: UndefElem);
5000 } else {
5001 SubVecAOps.push_back(Elt: UndefElem);
5002 SubVecBOps.push_back(Elt: Elem);
5003 }
5004 bool SelectMaskVal = (Idx < NumElts / 2);
5005 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
5006 }
5007 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
5008 MaskVals.size() == NumElts);
5009
5010 SDValue SubVecA = DAG.getBuildVector(VT, DL, Ops: SubVecAOps);
5011 SDValue SubVecB = DAG.getBuildVector(VT, DL, Ops: SubVecBOps);
5012 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
5013 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
5014 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: SubVecA, N3: SubVecB);
5015 }
5016
5017 // Cap the cost at a value linear to the number of elements in the vector.
5018 // The default lowering is to use the stack. The vector store + scalar loads
5019 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
5020 // being (at least) linear in LMUL. As a result, using the vslidedown
5021 // lowering for every element ends up being VL*LMUL..
5022 // TODO: Should we be directly costing the stack alternative? Doing so might
5023 // give us a more accurate upper bound.
5024 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
5025
5026 // TODO: unify with TTI getSlideCost.
5027 InstructionCost PerSlideCost = 1;
5028 switch (RISCVTargetLowering::getLMUL(VT: ContainerVT)) {
5029 default: break;
5030 case RISCVVType::LMUL_2:
5031 PerSlideCost = 2;
5032 break;
5033 case RISCVVType::LMUL_4:
5034 PerSlideCost = 4;
5035 break;
5036 case RISCVVType::LMUL_8:
5037 PerSlideCost = 8;
5038 break;
5039 }
5040
5041 // TODO: Should we be using the build instseq then cost + evaluate scheme
5042 // we use for integer constants here?
5043 unsigned UndefCount = 0;
5044 for (const SDValue &V : Op->ops()) {
5045 if (V.isUndef()) {
5046 UndefCount++;
5047 continue;
5048 }
5049 if (UndefCount) {
5050 LinearBudget -= PerSlideCost;
5051 UndefCount = 0;
5052 }
5053 LinearBudget -= PerSlideCost;
5054 }
5055 if (UndefCount) {
5056 LinearBudget -= PerSlideCost;
5057 }
5058
5059 if (LinearBudget < 0)
5060 return SDValue();
5061
5062 assert((!VT.isFloatingPoint() ||
5063 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
5064 "Illegal type which will result in reserved encoding");
5065
5066 const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
5067
5068 // General case: splat the first operand and slide other operands down one
5069 // by one to form a vector. Alternatively, if every operand is an
5070 // extraction from element 0 of a vector, we use that vector from the last
5071 // extraction as the start value and slide up instead of slide down. Such that
5072 // (1) we can avoid the initial splat (2) we can turn those vslide1up into
5073 // vslideup of 1 later and eliminate the vector to scalar movement, which is
5074 // something we cannot do with vslide1down/vslidedown.
5075 // Of course, using vslide1up/vslideup might increase the register pressure,
5076 // and that's why we conservatively limit to cases where every operand is an
5077 // extraction from the first element.
5078 SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
5079 SDValue EVec;
5080 bool SlideUp = false;
5081 auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
5082 SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
5083 if (SlideUp)
5084 return getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: Vec, Offset,
5085 Mask, VL, Policy);
5086 return getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: Vec, Offset,
5087 Mask, VL, Policy);
5088 };
5089
5090 // The reason we don't use all_of here is because we're also capturing EVec
5091 // from the last non-undef operand. If the std::execution_policy of the
5092 // underlying std::all_of is anything but std::sequenced_policy we might
5093 // capture the wrong EVec.
5094 for (SDValue V : Operands) {
5095 using namespace SDPatternMatch;
5096 SlideUp = V.isUndef() || sd_match(N: V, P: m_ExtractElt(Vec: m_Value(N&: EVec), Idx: m_Zero()));
5097 if (!SlideUp)
5098 break;
5099 }
5100
5101 // Do not slideup if the element type of EVec is different.
5102 if (SlideUp) {
5103 MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
5104 MVT ContainerEltVT = ContainerVT.getVectorElementType();
5105 if (EVecEltVT != ContainerEltVT)
5106 SlideUp = false;
5107 }
5108
5109 if (SlideUp) {
5110 MVT EVecContainerVT = EVec.getSimpleValueType();
5111 // Make sure the original vector has scalable vector type.
5112 if (EVecContainerVT.isFixedLengthVector()) {
5113 EVecContainerVT =
5114 getContainerForFixedLengthVector(VT: EVecContainerVT, Subtarget);
5115 EVec = convertToScalableVector(VT: EVecContainerVT, V: EVec, DAG, Subtarget);
5116 }
5117
5118 // Adapt EVec's type into ContainerVT.
5119 if (EVecContainerVT.getVectorMinNumElements() <
5120 ContainerVT.getVectorMinNumElements())
5121 EVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: EVec, Idx: 0);
5122 else
5123 EVec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec: EVec, Idx: 0);
5124
5125 // Reverse the elements as we're going to slide up from the last element.
5126 std::reverse(first: Operands.begin(), last: Operands.end());
5127 }
5128
5129 SDValue Vec;
5130 UndefCount = 0;
5131 for (SDValue V : Operands) {
5132 if (V.isUndef()) {
5133 UndefCount++;
5134 continue;
5135 }
5136
5137 // Start our sequence with either a TA splat or extract source in the
5138 // hopes that hardware is able to recognize there's no dependency on the
5139 // prior value of our temporary register.
5140 if (!Vec) {
5141 if (SlideUp) {
5142 Vec = EVec;
5143 } else {
5144 Vec = DAG.getSplatVector(VT, DL, Op: V);
5145 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
5146 }
5147
5148 UndefCount = 0;
5149 continue;
5150 }
5151
5152 if (UndefCount) {
5153 const SDValue Offset = DAG.getConstant(Val: UndefCount, DL, VT: Subtarget.getXLenVT());
5154 Vec = getVSlide(ContainerVT, DAG.getUNDEF(VT: ContainerVT), Vec, Offset, Mask,
5155 VL);
5156 UndefCount = 0;
5157 }
5158
5159 unsigned Opcode;
5160 if (VT.isFloatingPoint())
5161 Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
5162 else
5163 Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
5164
5165 if (!VT.isFloatingPoint())
5166 V = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: V);
5167 Vec = DAG.getNode(Opcode, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Vec,
5168 N3: V, N4: Mask, N5: VL);
5169 }
5170 if (UndefCount) {
5171 const SDValue Offset = DAG.getConstant(Val: UndefCount, DL, VT: Subtarget.getXLenVT());
5172 Vec = getVSlide(ContainerVT, DAG.getUNDEF(VT: ContainerVT), Vec, Offset, Mask,
5173 VL);
5174 }
5175 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5176}
5177
5178static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5179 SDValue Lo, SDValue Hi, SDValue VL,
5180 SelectionDAG &DAG) {
5181 if (!Passthru)
5182 Passthru = DAG.getUNDEF(VT);
5183 if (isa<ConstantSDNode>(Val: Lo) && isa<ConstantSDNode>(Val: Hi)) {
5184 int32_t LoC = cast<ConstantSDNode>(Val&: Lo)->getSExtValue();
5185 int32_t HiC = cast<ConstantSDNode>(Val&: Hi)->getSExtValue();
5186 // If Hi constant is all the same sign bit as Lo, lower this as a custom
5187 // node in order to try and match RVV vector/scalar instructions.
5188 if ((LoC >> 31) == HiC)
5189 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5190
5191 // Use vmv.v.x with EEW=32. Use either a vsetivli or vsetvli to change
5192 // VL. This can temporarily increase VL if VL less than VLMAX.
5193 if (LoC == HiC) {
5194 SDValue NewVL;
5195 if (isa<ConstantSDNode>(Val: VL) && isUInt<4>(x: VL->getAsZExtVal()))
5196 NewVL = DAG.getNode(Opcode: ISD::ADD, DL, VT: VL.getValueType(), N1: VL, N2: VL);
5197 else
5198 NewVL = DAG.getRegister(Reg: RISCV::X0, VT: MVT::i32);
5199 MVT InterVT =
5200 MVT::getVectorVT(VT: MVT::i32, EC: VT.getVectorElementCount() * 2);
5201 auto InterVec = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: InterVT,
5202 N1: DAG.getUNDEF(VT: InterVT), N2: Lo, N3: NewVL);
5203 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: InterVec);
5204 }
5205 }
5206
5207 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
5208 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(i: 0) == Lo &&
5209 isa<ConstantSDNode>(Val: Hi.getOperand(i: 1)) &&
5210 Hi.getConstantOperandVal(i: 1) == 31)
5211 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5212
5213 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
5214 // even if it might be sign extended.
5215 if (Hi.isUndef())
5216 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5217
5218 // Fall back to a stack store and stride x0 vector load.
5219 return DAG.getNode(Opcode: RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, N1: Passthru, N2: Lo,
5220 N3: Hi, N4: VL);
5221}
5222
5223// Called by type legalization to handle splat of i64 on RV32.
5224// FIXME: We can optimize this when the type has sign or zero bits in one
5225// of the halves.
5226static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5227 SDValue Scalar, SDValue VL,
5228 SelectionDAG &DAG) {
5229 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
5230 SDValue Lo, Hi;
5231 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Scalar, DL, LoVT: MVT::i32, HiVT: MVT::i32);
5232 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
5233}
5234
5235// This function lowers a splat of a scalar operand Splat with the vector
5236// length VL. It ensures the final sequence is type legal, which is useful when
5237// lowering a splat after type legalization.
5238static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
5239 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
5240 const RISCVSubtarget &Subtarget) {
5241 bool HasPassthru = Passthru && !Passthru.isUndef();
5242 if (!HasPassthru && !Passthru)
5243 Passthru = DAG.getUNDEF(VT);
5244
5245 MVT EltVT = VT.getVectorElementType();
5246 MVT XLenVT = Subtarget.getXLenVT();
5247
5248 if (VT.isFloatingPoint()) {
5249 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
5250 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
5251 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
5252 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
5253 Scalar = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Scalar);
5254 else
5255 Scalar = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Scalar);
5256 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
5257 Passthru = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Passthru);
5258 SDValue Splat =
5259 lowerScalarSplat(Passthru, Scalar, VL, VT: IVT, DL, DAG, Subtarget);
5260 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Splat);
5261 }
5262 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
5263 }
5264
5265 // Simplest case is that the operand needs to be promoted to XLenVT.
5266 if (Scalar.getValueType().bitsLE(VT: XLenVT)) {
5267 // If the operand is a constant, sign extend to increase our chances
5268 // of being able to use a .vi instruction. ANY_EXTEND would become a
5269 // a zero extend and the simm5 check in isel would fail.
5270 // FIXME: Should we ignore the upper bits in isel instead?
5271 unsigned ExtOpc =
5272 isa<ConstantSDNode>(Val: Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5273 Scalar = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: Scalar);
5274 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
5275 }
5276
5277 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
5278 "Unexpected scalar for splat lowering!");
5279
5280 if (isOneConstant(V: VL) && isNullConstant(V: Scalar))
5281 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: Passthru,
5282 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: VL);
5283
5284 // Otherwise use the more complicated splatting algorithm.
5285 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
5286}
5287
5288// This function lowers an insert of a scalar operand Scalar into lane
5289// 0 of the vector regardless of the value of VL. The contents of the
5290// remaining lanes of the result vector are unspecified. VL is assumed
5291// to be non-zero.
5292static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
5293 const SDLoc &DL, SelectionDAG &DAG,
5294 const RISCVSubtarget &Subtarget) {
5295 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
5296
5297 const MVT XLenVT = Subtarget.getXLenVT();
5298 SDValue Passthru = DAG.getUNDEF(VT);
5299
5300 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5301 isNullConstant(V: Scalar.getOperand(i: 1))) {
5302 SDValue ExtractedVal = Scalar.getOperand(i: 0);
5303 // The element types must be the same.
5304 if (ExtractedVal.getValueType().getVectorElementType() ==
5305 VT.getVectorElementType()) {
5306 MVT ExtractedVT = ExtractedVal.getSimpleValueType();
5307 MVT ExtractedContainerVT = ExtractedVT;
5308 if (ExtractedContainerVT.isFixedLengthVector()) {
5309 ExtractedContainerVT =
5310 getContainerForFixedLengthVector(VT: ExtractedContainerVT, Subtarget);
5311 ExtractedVal = convertToScalableVector(VT: ExtractedContainerVT,
5312 V: ExtractedVal, DAG, Subtarget);
5313 }
5314 if (ExtractedContainerVT.bitsLE(VT))
5315 return DAG.getInsertSubvector(DL, Vec: Passthru, SubVec: ExtractedVal, Idx: 0);
5316 return DAG.getExtractSubvector(DL, VT, Vec: ExtractedVal, Idx: 0);
5317 }
5318 }
5319
5320 if (VT.isFloatingPoint())
5321 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT, N1: DAG.getUNDEF(VT), N2: Scalar,
5322 N3: VL);
5323
5324 // Avoid the tricky legalization cases by falling back to using the
5325 // splat code which already handles it gracefully.
5326 if (!Scalar.getValueType().bitsLE(VT: XLenVT))
5327 return lowerScalarSplat(Passthru: DAG.getUNDEF(VT), Scalar,
5328 VL: DAG.getConstant(Val: 1, DL, VT: XLenVT),
5329 VT, DL, DAG, Subtarget);
5330
5331 // If the operand is a constant, sign extend to increase our chances
5332 // of being able to use a .vi instruction. ANY_EXTEND would become a
5333 // a zero extend and the simm5 check in isel would fail.
5334 // FIXME: Should we ignore the upper bits in isel instead?
5335 unsigned ExtOpc =
5336 isa<ConstantSDNode>(Val: Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5337 Scalar = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: Scalar);
5338 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: DAG.getUNDEF(VT), N2: Scalar,
5339 N3: VL);
5340}
5341
5342/// If concat_vector(V1,V2) could be folded away to some existing
5343/// vector source, return it. Note that the source may be larger
5344/// than the requested concat_vector (i.e. a extract_subvector
5345/// might be required.)
5346static SDValue foldConcatVector(SDValue V1, SDValue V2) {
5347 EVT VT = V1.getValueType();
5348 assert(VT == V2.getValueType() && "argument types must match");
5349 // Both input must be extracts.
5350 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5351 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
5352 return SDValue();
5353
5354 // Extracting from the same source.
5355 SDValue Src = V1.getOperand(i: 0);
5356 if (Src != V2.getOperand(i: 0) ||
5357 VT.isScalableVector() != Src.getValueType().isScalableVector())
5358 return SDValue();
5359
5360 // The extracts must extract the two halves of the source.
5361 if (V1.getConstantOperandVal(i: 1) != 0 ||
5362 V2.getConstantOperandVal(i: 1) != VT.getVectorMinNumElements())
5363 return SDValue();
5364
5365 return Src;
5366}
5367
5368// Can this shuffle be performed on exactly one (possibly larger) input?
5369static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
5370
5371 if (V2.isUndef())
5372 return V1;
5373
5374 unsigned NumElts = VT.getVectorNumElements();
5375 // Src needs to have twice the number of elements.
5376 // TODO: Update shuffle lowering to add the extract subvector
5377 if (SDValue Src = foldConcatVector(V1, V2);
5378 Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
5379 return Src;
5380
5381 return SDValue();
5382}
5383
5384static unsigned getLMULOctuple(MVT ContainerVT) {
5385 assert(ContainerVT.isScalableVector() && "Expected scalable vector type");
5386 unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
5387 assert(isPowerOf2_32(MinSize) && MinSize >= 8 && MinSize <= 512 &&
5388 "Unexpected LMUL");
5389 return MinSize / (RISCV::RVVBitsPerBlock / 8);
5390}
5391
5392static bool
5393isLegalVTForZvzipDeinterleavedOperand(MVT VT, const RISCVSubtarget &Subtarget) {
5394 MVT ContainerVT = VT;
5395 if (VT.isFixedLengthVector())
5396 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5397 return RISCVTargetLowering::getLMUL(VT: ContainerVT) != RISCVVType::LMUL_8;
5398}
5399
5400static bool
5401isLegalVTForZvzipInterleavedOperand(MVT VT, const RISCVSubtarget &Subtarget) {
5402 MVT ContainerVT = VT;
5403 if (VT.isFixedLengthVector()) {
5404 // VT is the interleaved result type, but lowerZvzipVZIP maps each
5405 // half-sized input to a scalable container before doubling that container
5406 // for the result. Mirror that order here because the minimum scalable
5407 // container size means mapping VT directly is not necessarily equivalent.
5408 // For example, interleaving two <2 x i32> inputs at VLEN=128 requires an
5409 // m2 result container, while mapping the <4 x i32> result directly yields
5410 // m1.
5411 ContainerVT = getContainerForFixedLengthVector(
5412 VT: VT.getHalfNumVectorElementsVT(), Subtarget);
5413 ContainerVT = ContainerVT.getDoubleNumVectorElementsVT();
5414 }
5415 unsigned EltBits = VT.getScalarSizeInBits();
5416 unsigned LMULOctuple = getLMULOctuple(ContainerVT);
5417 // Perform 2 * SEW <= LMUL * min(ELEN, VLEN) check.
5418 return EltBits * 16 <= LMULOctuple * std::min(a: Subtarget.getELen(),
5419 b: Subtarget.getRealMinVLen());
5420}
5421
5422/// Is this shuffle interleaving contiguous elements from one vector into the
5423/// even elements and contiguous elements from another vector into the odd
5424/// elements. \p EvenSrc will contain the element that should be in the first
5425/// even element. \p OddSrc will contain the element that should be in the first
5426/// odd element. These can be the first element in a source or the element half
5427/// way through the source.
5428static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
5429 int &OddSrc, const RISCVSubtarget &Subtarget) {
5430 // We need to be able to widen elements to the next larger integer type or
5431 // use the vzip instruction at e64.
5432 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
5433 if (!Subtarget.hasStdExtZvzip())
5434 return false;
5435 if (!isLegalVTForZvzipInterleavedOperand(VT, Subtarget))
5436 return false;
5437 }
5438
5439 int Size = Mask.size();
5440 int NumElts = VT.getVectorNumElements();
5441 assert(Size == (int)NumElts && "Unexpected mask size");
5442
5443 SmallVector<unsigned, 2> StartIndexes;
5444 if (!ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Size * 2, StartIndexes))
5445 return false;
5446
5447 EvenSrc = StartIndexes[0];
5448 OddSrc = StartIndexes[1];
5449
5450 // One source should be low half of first vector.
5451 if (EvenSrc != 0 && OddSrc != 0)
5452 return false;
5453
5454 // Subvectors will be subtracted from either at the start of the two input
5455 // vectors, or at the start and middle of the first vector if it's an unary
5456 // interleave.
5457 // In both cases, HalfNumElts will be extracted.
5458 // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
5459 // we'll create an illegal extract_subvector.
5460 // FIXME: We could support other values using a slidedown first.
5461 int HalfNumElts = NumElts / 2;
5462 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
5463}
5464
5465/// Is this mask representing a masked combination of two slides?
5466static bool isMaskedSlidePair(ArrayRef<int> Mask,
5467 std::array<std::pair<int, int>, 2> &SrcInfo) {
5468 if (!llvm::isMaskedSlidePair(Mask, NumElts: Mask.size(), SrcInfo))
5469 return false;
5470
5471 // Avoid matching vselect idioms
5472 if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
5473 return false;
5474 // Prefer vslideup as the second instruction, and identity
5475 // only as the initial instruction.
5476 if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
5477 SrcInfo[1].second == 0)
5478 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
5479 assert(SrcInfo[0].first != -1 && "Must find one slide");
5480 return true;
5481}
5482
5483// Exactly matches the semantics of a previously existing custom matcher
5484// to allow migration to new matcher without changing output.
5485static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
5486 unsigned NumElts) {
5487 if (SrcInfo[1].first == -1)
5488 return true;
5489 return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
5490 SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
5491}
5492
5493static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
5494 ArrayRef<int> Mask, unsigned Factor,
5495 bool RequiredPolarity) {
5496 int NumElts = Mask.size();
5497 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
5498 if (M < 0)
5499 continue;
5500 int Src = M >= NumElts;
5501 int Diff = (int)Idx - (M % NumElts);
5502 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
5503 assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
5504 "Must match exactly one of the two slides");
5505 if (RequiredPolarity != (C == (Idx / Factor) % 2))
5506 return false;
5507 }
5508 return true;
5509}
5510
5511/// Given a shuffle which can be represented as a pair of two slides,
5512/// see if it is a pair-even idiom.
5513/// Pair-even is:
5514/// vs2: a0 a1 a2 a3
5515/// vs1: b0 b1 b2 b3
5516/// vd: a0 b0 a2 b2
5517static bool isPairEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
5518 ArrayRef<int> Mask, unsigned &Factor) {
5519 Factor = SrcInfo[1].second;
5520 return SrcInfo[0].second == 0 && isPowerOf2_32(Value: Factor) &&
5521 Mask.size() % Factor == 0 &&
5522 isAlternating(SrcInfo, Mask, Factor, RequiredPolarity: true);
5523}
5524
5525/// Given a shuffle which can be represented as a pair of two slides,
5526/// see if it is a pair-odd idiom.
5527/// Pair-odd is:
5528/// vs2: a0 a1 a2 a3
5529/// vs1: b0 b1 b2 b3
5530/// vd: a1 b1 a3 b3
5531/// Note that the operand order is swapped due to the way we canonicalize
5532/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
5533static bool isPairOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
5534 ArrayRef<int> Mask, unsigned &Factor) {
5535 Factor = -SrcInfo[1].second;
5536 return SrcInfo[0].second == 0 && isPowerOf2_32(Value: Factor) &&
5537 Mask.size() % Factor == 0 &&
5538 isAlternating(SrcInfo, Mask, Factor, RequiredPolarity: false);
5539}
5540
5541// Lower a deinterleave shuffle to SRL and TRUNC. Factor must be
5542// 2, 4, 8 and the integer type Factor-times larger than VT's
5543// element type must be a legal element type.
5544// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
5545// -> [p, q, r, s] (Factor=2, Index=1)
5546static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
5547 SDValue Src, unsigned Factor,
5548 unsigned Index, SelectionDAG &DAG) {
5549 unsigned EltBits = VT.getScalarSizeInBits();
5550 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
5551 MVT WideSrcVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits * Factor),
5552 EC: SrcEC.divideCoefficientBy(RHS: Factor));
5553 MVT ResVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits),
5554 EC: SrcEC.divideCoefficientBy(RHS: Factor));
5555 Src = DAG.getBitcast(VT: WideSrcVT, V: Src);
5556
5557 unsigned Shift = Index * EltBits;
5558 SDValue Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: WideSrcVT, N1: Src,
5559 N2: DAG.getConstant(Val: Shift, DL, VT: WideSrcVT));
5560 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResVT, Operand: Res);
5561 MVT CastVT = ResVT.changeVectorElementType(EltVT: VT.getVectorElementType());
5562 Res = DAG.getBitcast(VT: CastVT, V: Res);
5563 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: Res, Idx: 0);
5564}
5565
5566/// Match a single source shuffle which is an identity except that some
5567/// particular element is repeated. This can be lowered as a masked
5568/// vrgather.vi/vx. Note that the two source form of this is handled
5569/// by the recursive splitting logic and doesn't need special handling.
5570static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
5571 const RISCVSubtarget &Subtarget,
5572 SelectionDAG &DAG) {
5573
5574 SDLoc DL(SVN);
5575 MVT VT = SVN->getSimpleValueType(ResNo: 0);
5576 SDValue V1 = SVN->getOperand(Num: 0);
5577 assert(SVN->getOperand(1).isUndef());
5578 ArrayRef<int> Mask = SVN->getMask();
5579 const unsigned NumElts = VT.getVectorNumElements();
5580 MVT XLenVT = Subtarget.getXLenVT();
5581
5582 std::optional<int> SplatIdx;
5583 for (auto [I, M] : enumerate(First&: Mask)) {
5584 if (M == -1 || I == (unsigned)M)
5585 continue;
5586 if (SplatIdx && *SplatIdx != M)
5587 return SDValue();
5588 SplatIdx = M;
5589 }
5590
5591 if (!SplatIdx)
5592 return SDValue();
5593
5594 SmallVector<SDValue> MaskVals;
5595 for (int MaskIndex : Mask) {
5596 bool SelectMaskVal = MaskIndex == *SplatIdx;
5597 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
5598 }
5599 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5600 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
5601 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
5602 SDValue Splat = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT),
5603 Mask: SmallVector<int>(NumElts, *SplatIdx));
5604 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: Splat, N3: V1);
5605}
5606
5607// Lower the following shuffle to vslidedown.
5608// a)
5609// t49: v8i8 = extract_subvector t13, Constant:i64<0>
5610// t109: v8i8 = extract_subvector t13, Constant:i64<8>
5611// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
5612// b)
5613// t69: v16i16 = extract_subvector t68, Constant:i64<0>
5614// t23: v8i16 = extract_subvector t69, Constant:i64<0>
5615// t29: v4i16 = extract_subvector t23, Constant:i64<4>
5616// t26: v8i16 = extract_subvector t69, Constant:i64<8>
5617// t30: v4i16 = extract_subvector t26, Constant:i64<0>
5618// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
5619static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
5620 SDValue V1, SDValue V2,
5621 ArrayRef<int> Mask,
5622 const RISCVSubtarget &Subtarget,
5623 SelectionDAG &DAG) {
5624 auto findNonEXTRACT_SUBVECTORParent =
5625 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
5626 uint64_t Offset = 0;
5627 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5628 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
5629 // a scalable vector. But we don't want to match the case.
5630 Parent.getOperand(i: 0).getSimpleValueType().isFixedLengthVector()) {
5631 Offset += Parent.getConstantOperandVal(i: 1);
5632 Parent = Parent.getOperand(i: 0);
5633 }
5634 return std::make_pair(x&: Parent, y&: Offset);
5635 };
5636
5637 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
5638 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
5639
5640 // Extracting from the same source.
5641 SDValue Src = V1Src;
5642 if (Src != V2Src)
5643 return SDValue();
5644
5645 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
5646 SmallVector<int, 16> NewMask(Mask);
5647 for (size_t i = 0; i != NewMask.size(); ++i) {
5648 if (NewMask[i] == -1)
5649 continue;
5650
5651 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
5652 NewMask[i] = NewMask[i] + V1IndexOffset;
5653 } else {
5654 // Minus NewMask.size() is needed. Otherwise, the b case would be
5655 // <5,6,7,12> instead of <5,6,7,8>.
5656 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
5657 }
5658 }
5659
5660 // First index must be known and non-zero. It will be used as the slidedown
5661 // amount.
5662 if (NewMask[0] <= 0)
5663 return SDValue();
5664
5665 // NewMask is also continuous.
5666 for (unsigned i = 1; i != NewMask.size(); ++i)
5667 if (NewMask[i - 1] + 1 != NewMask[i])
5668 return SDValue();
5669
5670 MVT XLenVT = Subtarget.getXLenVT();
5671 MVT SrcVT = Src.getSimpleValueType();
5672 MVT ContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
5673 auto [TrueMask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
5674 SDValue Slidedown =
5675 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru: DAG.getUNDEF(VT: ContainerVT),
5676 Op: convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget),
5677 Offset: DAG.getConstant(Val: NewMask[0], DL, VT: XLenVT), Mask: TrueMask, VL);
5678 return DAG.getExtractSubvector(
5679 DL, VT, Vec: convertFromScalableVector(VT: SrcVT, V: Slidedown, DAG, Subtarget), Idx: 0);
5680}
5681
5682// Because vslideup leaves the destination elements at the start intact, we can
5683// use it to perform shuffles that insert subvectors:
5684//
5685// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
5686// ->
5687// vsetvli zero, 8, e8, mf2, ta, ma
5688// vslideup.vi v8, v9, 4
5689//
5690// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
5691// ->
5692// vsetvli zero, 5, e8, mf2, tu, ma
5693// vslideup.v1 v8, v9, 2
5694static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
5695 SDValue V1, SDValue V2,
5696 ArrayRef<int> Mask,
5697 const RISCVSubtarget &Subtarget,
5698 SelectionDAG &DAG) {
5699 unsigned NumElts = VT.getVectorNumElements();
5700 int NumSubElts, Index;
5701 if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumSrcElts: NumElts, NumSubElts,
5702 Index))
5703 return SDValue();
5704
5705 bool OpsSwapped = Mask[Index] < (int)NumElts;
5706 SDValue InPlace = OpsSwapped ? V2 : V1;
5707 SDValue ToInsert = OpsSwapped ? V1 : V2;
5708
5709 MVT XLenVT = Subtarget.getXLenVT();
5710 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5711 auto TrueMask = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).first;
5712 // We slide up by the index that the subvector is being inserted at, and set
5713 // VL to the index + the number of elements being inserted.
5714 unsigned Policy =
5715 RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
5716 // If the we're adding a suffix to the in place vector, i.e. inserting right
5717 // up to the very end of it, then we don't actually care about the tail.
5718 if (NumSubElts + Index >= (int)NumElts)
5719 Policy |= RISCVVType::TAIL_AGNOSTIC;
5720
5721 InPlace = convertToScalableVector(VT: ContainerVT, V: InPlace, DAG, Subtarget);
5722 ToInsert = convertToScalableVector(VT: ContainerVT, V: ToInsert, DAG, Subtarget);
5723 SDValue VL = DAG.getConstant(Val: NumSubElts + Index, DL, VT: XLenVT);
5724
5725 SDValue Res;
5726 // If we're inserting into the lowest elements, use a tail undisturbed
5727 // vmv.v.v.
5728 if (Index == 0)
5729 Res = DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: ContainerVT, N1: InPlace, N2: ToInsert,
5730 N3: VL);
5731 else
5732 Res = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: InPlace, Op: ToInsert,
5733 Offset: DAG.getConstant(Val: Index, DL, VT: XLenVT), Mask: TrueMask, VL, Policy);
5734 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
5735}
5736
5737// A shuffle of shuffles where the final data only is drawn from 2 input ops
5738// can be compressed into a single shuffle
5739static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
5740 const RISCVSubtarget &Subtarget,
5741 SelectionDAG &DAG) {
5742 SDValue V1 = SVN->getOperand(Num: 0);
5743 SDValue V2 = SVN->getOperand(Num: 1);
5744
5745 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
5746 V2.getOpcode() != ISD::VECTOR_SHUFFLE)
5747 return SDValue();
5748
5749 if (!V1.hasOneUse() || !V2.hasOneUse())
5750 return SDValue();
5751
5752 ArrayRef<int> Mask = SVN->getMask();
5753 ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(Val: V1.getNode())->getMask();
5754 ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(Val: V2.getNode())->getMask();
5755 unsigned NumElts = Mask.size();
5756 SmallVector<int> NewMask(NumElts, -1);
5757 for (unsigned Idx : seq<unsigned>(Size: NumElts)) {
5758 int Lane = Mask[Idx];
5759 // Don't assign if poison
5760 if (Lane == -1)
5761 continue;
5762 int OrigLane;
5763 bool SecondOp = false;
5764 if ((unsigned)Lane < NumElts) {
5765 OrigLane = V1Mask[Lane];
5766 } else {
5767 OrigLane = V2Mask[Lane - NumElts];
5768 SecondOp = true;
5769 }
5770 if (OrigLane == -1)
5771 continue;
5772 // Don't handle if shuffling from a second operand
5773 if ((unsigned)OrigLane >= NumElts)
5774 return SDValue();
5775 if (SecondOp)
5776 OrigLane += NumElts;
5777 NewMask[Idx] = OrigLane;
5778 }
5779
5780 EVT VT = SVN->getValueType(ResNo: 0);
5781 SDLoc DL(SVN);
5782
5783 return DAG.getVectorShuffle(VT, dl: DL, N1: V1->getOperand(Num: 0), N2: V2->getOperand(Num: 0),
5784 Mask: NewMask);
5785}
5786
5787/// Match v(f)slide1up/down idioms. These operations involve sliding
5788/// N-1 elements to make room for an inserted scalar at one end.
5789static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
5790 SDValue V1, SDValue V2,
5791 ArrayRef<int> Mask,
5792 const RISCVSubtarget &Subtarget,
5793 SelectionDAG &DAG) {
5794 bool OpsSwapped = false;
5795 if (!isa<BuildVectorSDNode>(Val: V1)) {
5796 if (!isa<BuildVectorSDNode>(Val: V2))
5797 return SDValue();
5798 std::swap(a&: V1, b&: V2);
5799 OpsSwapped = true;
5800 }
5801 SDValue Splat = cast<BuildVectorSDNode>(Val&: V1)->getSplatValue();
5802 if (!Splat)
5803 return SDValue();
5804
5805 // Return true if the mask could describe a slide of Mask.size() - 1
5806 // elements from concat_vector(V1, V2)[Base:] to [Offset:].
5807 auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
5808 const unsigned S = (Offset > 0) ? 0 : -Offset;
5809 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
5810 for (unsigned i = S; i != E; ++i)
5811 if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
5812 return false;
5813 return true;
5814 };
5815
5816 const unsigned NumElts = VT.getVectorNumElements();
5817 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
5818 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
5819 return SDValue();
5820
5821 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
5822 // Inserted lane must come from splat, undef scalar is legal but not profitable.
5823 if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
5824 return SDValue();
5825
5826 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5827 auto [TrueMask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
5828
5829 // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
5830 // vslide1{down,up}.vx instead.
5831 if ((VT.getVectorElementType() == MVT::bf16 &&
5832 !Subtarget.hasVInstructionsBF16()) ||
5833 (VT.getVectorElementType() == MVT::f16 &&
5834 !Subtarget.hasVInstructionsF16())) {
5835 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
5836 Splat =
5837 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: Subtarget.getXLenVT(), Operand: Splat);
5838 V2 = DAG.getBitcast(
5839 VT: IntVT, V: convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget));
5840 SDValue Vec = DAG.getNode(
5841 Opcode: IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
5842 VT: IntVT, N1: DAG.getUNDEF(VT: IntVT), N2: V2, N3: Splat, N4: TrueMask, N5: VL);
5843 Vec = DAG.getBitcast(VT: ContainerVT, V: Vec);
5844 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5845 }
5846
5847 auto OpCode = IsVSlidedown ?
5848 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
5849 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
5850 if (!VT.isFloatingPoint())
5851 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: Splat);
5852 auto Vec = DAG.getNode(Opcode: OpCode, DL, VT: ContainerVT,
5853 N1: DAG.getUNDEF(VT: ContainerVT),
5854 N2: convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget),
5855 N3: Splat, N4: TrueMask, N5: VL);
5856 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5857}
5858
5859/// Match a mask which "spreads" the leading elements of a vector evenly
5860/// across the result. Factor is the spread amount, and Index is the
5861/// offset applied. (on success, Index < Factor) This is the inverse
5862/// of a deinterleave with the same Factor and Index. This is analogous
5863/// to an interleave, except that all but one lane is undef.
5864bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
5865 unsigned &Index) {
5866 SmallVector<bool> LaneIsUndef(Factor, true);
5867 for (unsigned i = 0; i < Mask.size(); i++)
5868 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
5869
5870 bool Found = false;
5871 for (unsigned i = 0; i < Factor; i++) {
5872 if (LaneIsUndef[i])
5873 continue;
5874 if (Found)
5875 return false;
5876 Index = i;
5877 Found = true;
5878 }
5879 if (!Found)
5880 return false;
5881
5882 for (unsigned i = 0; i < Mask.size() / Factor; i++) {
5883 unsigned j = i * Factor + Index;
5884 if (Mask[j] != -1 && (unsigned)Mask[j] != i)
5885 return false;
5886 }
5887 return true;
5888}
5889
5890static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
5891 const SDLoc &DL, SelectionDAG &DAG,
5892 const RISCVSubtarget &Subtarget) {
5893 assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
5894 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5895
5896 MVT VT = Op0.getSimpleValueType();
5897 MVT IntVT = VT.changeVectorElementTypeToInteger();
5898 Op0 = DAG.getBitcast(VT: IntVT, V: Op0);
5899 Op1 = DAG.getBitcast(VT: IntVT, V: Op1);
5900
5901 MVT ContainerVT = IntVT;
5902 if (VT.isFixedLengthVector()) {
5903 ContainerVT = getContainerForFixedLengthVector(VT: IntVT, Subtarget);
5904 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
5905 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
5906 }
5907
5908 MVT InnerVT = ContainerVT;
5909 auto [Mask, VL] = getDefaultVLOps(VecVT: IntVT, ContainerVT: InnerVT, DL, DAG, Subtarget);
5910
5911 SDValue Passthru = DAG.getUNDEF(VT: InnerVT);
5912 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: InnerVT, N1: Op0, N2: Op1, N3: Passthru, N4: Mask, N5: VL);
5913 if (IntVT.isFixedLengthVector())
5914 Res = convertFromScalableVector(VT: IntVT, V: Res, DAG, Subtarget);
5915 Res = DAG.getBitcast(VT, V: Res);
5916 return Res;
5917}
5918
5919static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
5920 SelectionDAG &DAG,
5921 const RISCVSubtarget &Subtarget) {
5922 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5923 MVT VT = Op0.getSimpleValueType();
5924 MVT IntVT = VT.changeVectorElementTypeToInteger();
5925 Op0 = DAG.getBitcast(VT: IntVT, V: Op0);
5926 Op1 = DAG.getBitcast(VT: IntVT, V: Op1);
5927 MVT ContainerVT = IntVT;
5928 if (VT.isFixedLengthVector()) {
5929 ContainerVT = getContainerForFixedLengthVector(VT: IntVT, Subtarget);
5930 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
5931 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
5932 }
5933 MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
5934 MVT ResIntVT = IntVT.getDoubleNumVectorElementsVT();
5935 auto [Mask, VL] = getDefaultVLOps(VecVT: ResIntVT, ContainerVT: ResVT, DL, DAG, Subtarget);
5936 SDValue Passthru = DAG.getUNDEF(VT: ResVT);
5937 SDValue Res =
5938 DAG.getNode(Opcode: RISCVISD::VZIP_VL, DL, VT: ResVT, N1: Op0, N2: Op1, N3: Passthru, N4: Mask, N5: VL);
5939 if (IntVT.isFixedLengthVector())
5940 Res = convertFromScalableVector(VT: IntVT.getDoubleNumVectorElementsVT(), V: Res,
5941 DAG, Subtarget);
5942 Res = DAG.getBitcast(VT: VT.getDoubleNumVectorElementsVT(), V: Res);
5943 return Res;
5944}
5945
5946static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
5947 SelectionDAG &DAG,
5948 const RISCVSubtarget &Subtarget) {
5949 assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
5950 MVT VT = Op.getSimpleValueType();
5951 assert(VT.getVectorMinNumElements() >= 2);
5952
5953 MVT IntVT = VT.changeVectorElementTypeToInteger();
5954 Op = DAG.getBitcast(VT: IntVT, V: Op);
5955 MVT ContainerVT = IntVT;
5956 if (VT.isFixedLengthVector()) {
5957 ContainerVT = getContainerForFixedLengthVector(VT: IntVT, Subtarget);
5958 // Ensure that halving the source container produces a legal result type.
5959 // In particular, E64 has no fractional LMUL types, and some Zve profiles
5960 // do not legalize the smallest fractional LMUL type.
5961 if (ContainerVT.getVectorMinNumElements() == 1 ||
5962 !Subtarget.getTargetLowering()->isTypeLegal(
5963 VT: ContainerVT.getHalfNumVectorElementsVT()))
5964 ContainerVT = ContainerVT.getDoubleNumVectorElementsVT();
5965 Op = convertToScalableVector(VT: ContainerVT, V: Op, DAG, Subtarget);
5966 }
5967
5968 MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
5969 MVT HalfVT = VT.getHalfNumVectorElementsVT();
5970 MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
5971 SDValue VL = getDefaultVLOps(VecVT: HalfIntVT, ContainerVT: ResVT, DL, DAG, Subtarget).second;
5972 SDValue Passthru = DAG.getUNDEF(VT: ResVT);
5973 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ResVT, N1: Op, N2: Passthru, N3: VL);
5974 if (HalfIntVT.isFixedLengthVector())
5975 Res = convertFromScalableVector(VT: HalfIntVT, V: Res, DAG, Subtarget);
5976 Res = DAG.getBitcast(VT: HalfVT, V: Res);
5977 return Res;
5978}
5979
5980// Given a vector a, b, c, d return a vector Factor times longer
5981// with Factor-1 undef's between elements. Ex:
5982// a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
5983// undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
5984static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
5985 const SDLoc &DL, SelectionDAG &DAG) {
5986
5987 MVT VT = V.getSimpleValueType();
5988 unsigned EltBits = VT.getScalarSizeInBits();
5989 ElementCount EC = VT.getVectorElementCount();
5990 V = DAG.getBitcast(VT: VT.changeTypeToInteger(), V);
5991
5992 MVT WideVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits * Factor), EC);
5993
5994 SDValue Result = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: V);
5995 // TODO: On rv32, the constant becomes a splat_vector_parts which does not
5996 // allow the SHL to fold away if Index is 0.
5997 if (Index != 0)
5998 Result = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Result,
5999 N2: DAG.getConstant(Val: EltBits * Index, DL, VT: WideVT));
6000 // Make sure to use original element type
6001 MVT ResultVT = MVT::getVectorVT(VT: VT.getVectorElementType(), EC: EC * Factor);
6002 return DAG.getBitcast(VT: ResultVT, V: Result);
6003}
6004
6005// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
6006// to create an interleaved vector of <[vscale x] n*2 x ty>.
6007// This requires that the size of ty is less than the subtarget's maximum ELEN.
6008static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
6009 const SDLoc &DL, SelectionDAG &DAG,
6010 const RISCVSubtarget &Subtarget) {
6011
6012 // FIXME: Not only does this optimize the code, it fixes some correctness
6013 // issues because MIR does not have freeze.
6014 if (EvenV.isUndef())
6015 return getWideningSpread(V: OddV, Factor: 2, Index: 1, DL, DAG);
6016 if (OddV.isUndef())
6017 return getWideningSpread(V: EvenV, Factor: 2, Index: 0, DL, DAG);
6018
6019 MVT VecVT = EvenV.getSimpleValueType();
6020 MVT VecContainerVT = VecVT; // <vscale x n x ty>
6021 // Convert fixed vectors to scalable if needed
6022 if (VecContainerVT.isFixedLengthVector()) {
6023 VecContainerVT = getContainerForFixedLengthVector(VT: VecVT, Subtarget);
6024 EvenV = convertToScalableVector(VT: VecContainerVT, V: EvenV, DAG, Subtarget);
6025 OddV = convertToScalableVector(VT: VecContainerVT, V: OddV, DAG, Subtarget);
6026 }
6027
6028 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
6029
6030 // We're working with a vector of the same size as the resulting
6031 // interleaved vector, but with half the number of elements and
6032 // twice the SEW (Hence the restriction on not using the maximum
6033 // ELEN)
6034 MVT WideVT =
6035 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VecVT.getScalarSizeInBits() * 2),
6036 EC: VecVT.getVectorElementCount());
6037 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
6038 if (WideContainerVT.isFixedLengthVector())
6039 WideContainerVT = getContainerForFixedLengthVector(VT: WideVT, Subtarget);
6040
6041 // Bitcast the input vectors to integers in case they are FP
6042 VecContainerVT = VecContainerVT.changeTypeToInteger();
6043 EvenV = DAG.getBitcast(VT: VecContainerVT, V: EvenV);
6044 OddV = DAG.getBitcast(VT: VecContainerVT, V: OddV);
6045
6046 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: VecContainerVT, DL, DAG, Subtarget);
6047 SDValue Passthru = DAG.getUNDEF(VT: WideContainerVT);
6048
6049 SDValue Interleaved;
6050 if (Subtarget.hasStdExtZvbb()) {
6051 // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
6052 SDValue OffsetVec =
6053 DAG.getConstant(Val: VecVT.getScalarSizeInBits(), DL, VT: VecContainerVT);
6054 Interleaved = DAG.getNode(Opcode: RISCVISD::VWSLL_VL, DL, VT: WideContainerVT, N1: OddV,
6055 N2: OffsetVec, N3: Passthru, N4: Mask, N5: VL);
6056 Interleaved = DAG.getNode(Opcode: RISCVISD::VWADDU_W_VL, DL, VT: WideContainerVT,
6057 N1: Interleaved, N2: EvenV, N3: Passthru, N4: Mask, N5: VL);
6058 } else {
6059 // FIXME: We should freeze the odd vector here. We already handled the case
6060 // of provably undef/poison above.
6061
6062 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
6063 // vwaddu.vv
6064 Interleaved = DAG.getNode(Opcode: RISCVISD::VWADDU_VL, DL, VT: WideContainerVT, N1: EvenV,
6065 N2: OddV, N3: Passthru, N4: Mask, N5: VL);
6066
6067 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
6068 SDValue AllOnesVec = DAG.getSplatVector(
6069 VT: VecContainerVT, DL, Op: DAG.getAllOnesConstant(DL, VT: Subtarget.getXLenVT()));
6070 SDValue OddsMul = DAG.getNode(Opcode: RISCVISD::VWMULU_VL, DL, VT: WideContainerVT,
6071 N1: OddV, N2: AllOnesVec, N3: Passthru, N4: Mask, N5: VL);
6072
6073 // Add the two together so we get
6074 // (OddV * 0xff...ff) + (OddV + EvenV)
6075 // = (OddV * 0x100...00) + EvenV
6076 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
6077 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
6078 Interleaved = DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT: WideContainerVT,
6079 N1: Interleaved, N2: OddsMul, N3: Passthru, N4: Mask, N5: VL);
6080 }
6081
6082 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
6083 MVT ResultContainerVT = MVT::getVectorVT(
6084 VT: VecVT.getVectorElementType(), // Make sure to use original type
6085 EC: VecContainerVT.getVectorElementCount() * 2);
6086 Interleaved = DAG.getBitcast(VT: ResultContainerVT, V: Interleaved);
6087
6088 // Convert back to a fixed vector if needed
6089 MVT ResultVT = MVT::getVectorVT(VT: VecVT.getVectorElementType(),
6090 EC: VecVT.getVectorElementCount() * 2);
6091 if (ResultVT.isFixedLengthVector())
6092 Interleaved =
6093 convertFromScalableVector(VT: ResultVT, V: Interleaved, DAG, Subtarget);
6094
6095 return Interleaved;
6096}
6097
6098// If we have a vector of bits that we want to reverse, we can use a vbrev on a
6099// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
6100static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
6101 SelectionDAG &DAG,
6102 const RISCVSubtarget &Subtarget) {
6103 SDLoc DL(SVN);
6104 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6105 SDValue V = SVN->getOperand(Num: 0);
6106 unsigned NumElts = VT.getVectorNumElements();
6107
6108 assert(VT.getVectorElementType() == MVT::i1);
6109
6110 if (!ShuffleVectorInst::isReverseMask(Mask: SVN->getMask(),
6111 NumSrcElts: SVN->getMask().size()) ||
6112 !SVN->getOperand(Num: 1).isUndef())
6113 return SDValue();
6114
6115 unsigned ViaEltSize = std::max(a: (uint64_t)8, b: PowerOf2Ceil(A: NumElts));
6116 EVT ViaVT = EVT::getVectorVT(
6117 Context&: *DAG.getContext(), VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ViaEltSize), NumElements: 1);
6118 EVT ViaBitVT =
6119 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: ViaVT.getScalarSizeInBits());
6120
6121 // If we don't have zvbb or the larger element type > ELEN, the operation will
6122 // be illegal.
6123 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(Op: ISD::BITREVERSE,
6124 VT: ViaVT) ||
6125 !Subtarget.getTargetLowering()->isTypeLegal(VT: ViaBitVT))
6126 return SDValue();
6127
6128 // If the bit vector doesn't fit exactly into the larger element type, we need
6129 // to insert it into the larger vector and then shift up the reversed bits
6130 // afterwards to get rid of the gap introduced.
6131 if (ViaEltSize > NumElts)
6132 V = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ViaBitVT), SubVec: V, Idx: 0);
6133
6134 SDValue Res =
6135 DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ViaVT, Operand: DAG.getBitcast(VT: ViaVT, V));
6136
6137 // Shift up the reversed bits if the vector didn't exactly fit into the larger
6138 // element type.
6139 if (ViaEltSize > NumElts)
6140 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: ViaVT, N1: Res,
6141 N2: DAG.getConstant(Val: ViaEltSize - NumElts, DL, VT: ViaVT));
6142
6143 Res = DAG.getBitcast(VT: ViaBitVT, V: Res);
6144
6145 if (ViaEltSize > NumElts)
6146 Res = DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0);
6147 return Res;
6148}
6149
6150static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
6151 const RISCVSubtarget &Subtarget,
6152 MVT &RotateVT, unsigned &RotateAmt) {
6153 unsigned NumElts = VT.getVectorNumElements();
6154 unsigned EltSizeInBits = VT.getScalarSizeInBits();
6155 unsigned NumSubElts;
6156 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, MinSubElts: 2,
6157 MaxSubElts: NumElts, NumSubElts, RotateAmt))
6158 return false;
6159 RotateVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSizeInBits * NumSubElts),
6160 NumElements: NumElts / NumSubElts);
6161
6162 // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
6163 return Subtarget.getTargetLowering()->isTypeLegal(VT: RotateVT);
6164}
6165
6166// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
6167// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
6168// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
6169static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
6170 SelectionDAG &DAG,
6171 const RISCVSubtarget &Subtarget) {
6172 SDLoc DL(SVN);
6173
6174 EVT VT = SVN->getValueType(ResNo: 0);
6175 unsigned RotateAmt;
6176 MVT RotateVT;
6177 if (!isLegalBitRotate(Mask: SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
6178 return SDValue();
6179
6180 SDValue Op = DAG.getBitcast(VT: RotateVT, V: SVN->getOperand(Num: 0));
6181
6182 SDValue Rotate;
6183 // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
6184 // so canonicalize to vrev8.
6185 if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
6186 Rotate = DAG.getNode(Opcode: ISD::BSWAP, DL, VT: RotateVT, Operand: Op);
6187 else
6188 Rotate = DAG.getNode(Opcode: ISD::ROTL, DL, VT: RotateVT, N1: Op,
6189 N2: DAG.getConstant(Val: RotateAmt, DL, VT: RotateVT));
6190
6191 return DAG.getBitcast(VT, V: Rotate);
6192}
6193
6194// If compiling with an exactly known VLEN, see if we can split a
6195// shuffle on m2 or larger into a small number of m1 sized shuffles
6196// which write each destination registers exactly once.
6197static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
6198 SelectionDAG &DAG,
6199 const RISCVSubtarget &Subtarget) {
6200 SDLoc DL(SVN);
6201 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6202 SDValue V1 = SVN->getOperand(Num: 0);
6203 SDValue V2 = SVN->getOperand(Num: 1);
6204 ArrayRef<int> Mask = SVN->getMask();
6205
6206 // If we don't know exact data layout, not much we can do. If this
6207 // is already m1 or smaller, no point in splitting further.
6208 const auto VLen = Subtarget.getRealVLen();
6209 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
6210 return SDValue();
6211
6212 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
6213 // expansion for.
6214 unsigned RotateAmt;
6215 MVT RotateVT;
6216 if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
6217 return SDValue();
6218
6219 MVT ElemVT = VT.getVectorElementType();
6220 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
6221
6222 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
6223 MVT OneRegVT = MVT::getVectorVT(VT: ElemVT, NumElements: ElemsPerVReg);
6224 MVT M1VT = getContainerForFixedLengthVector(VT: OneRegVT, Subtarget);
6225 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
6226 unsigned NumOpElts = M1VT.getVectorMinNumElements();
6227 unsigned NumElts = ContainerVT.getVectorMinNumElements();
6228 unsigned NumOfSrcRegs = NumElts / NumOpElts;
6229 unsigned NumOfDestRegs = NumElts / NumOpElts;
6230 // The following semantically builds up a fixed length concat_vector
6231 // of the component shuffle_vectors. We eagerly lower to scalable here
6232 // to avoid DAG combining it back to a large shuffle_vector again.
6233 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
6234 V2 = convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget);
6235 SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
6236 Operands;
6237 processShuffleMasks(
6238 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs,
6239 NoInputAction: [&]() { Operands.emplace_back(); },
6240 SingleInputAction: [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
6241 Operands.emplace_back().emplace_back(Args&: SrcVecIdx, UINT_MAX,
6242 Args: SmallVector<int>(SrcSubMask));
6243 },
6244 ManyInputsAction: [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
6245 if (NewReg)
6246 Operands.emplace_back();
6247 Operands.back().emplace_back(Args&: Idx1, Args&: Idx2, Args: SmallVector<int>(SrcSubMask));
6248 });
6249 assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
6250 // Note: check that we do not emit too many shuffles here to prevent code
6251 // size explosion.
6252 // TODO: investigate, if it can be improved by extra analysis of the masks to
6253 // check if the code is more profitable.
6254 unsigned NumShuffles = std::accumulate(
6255 first: Operands.begin(), last: Operands.end(), init: 0u,
6256 binary_op: [&](unsigned N,
6257 ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
6258 if (Data.empty())
6259 return N;
6260 N += Data.size();
6261 for (const auto &P : Data) {
6262 unsigned Idx2 = std::get<1>(t: P);
6263 ArrayRef<int> Mask = std::get<2>(t: P);
6264 if (Idx2 != UINT_MAX)
6265 ++N;
6266 else if (ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts: Mask.size()))
6267 --N;
6268 }
6269 return N;
6270 });
6271 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
6272 (NumOfDestRegs <= 2 && NumShuffles >= 4))
6273 return SDValue();
6274 auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
6275 SDValue SubVec = DAG.getExtractSubvector(DL, VT: M1VT, Vec: SrcVec, Idx: ExtractIdx);
6276 SubVec = convertFromScalableVector(VT: OneRegVT, V: SubVec, DAG, Subtarget);
6277 return SubVec;
6278 };
6279 auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
6280 ArrayRef<int> Mask) {
6281 SDValue SubVec = DAG.getVectorShuffle(VT: OneRegVT, dl: DL, N1: SubVec1, N2: SubVec2, Mask);
6282 return SubVec;
6283 };
6284 SDValue Vec = DAG.getUNDEF(VT: ContainerVT);
6285 for (auto [I, Data] : enumerate(First&: Operands)) {
6286 if (Data.empty())
6287 continue;
6288 SmallDenseMap<unsigned, SDValue, 4> Values;
6289 for (unsigned I : seq<unsigned>(Size: Data.size())) {
6290 const auto &[Idx1, Idx2, _] = Data[I];
6291 // If the shuffle contains permutation of odd number of elements,
6292 // Idx1 might be used already in the first iteration.
6293 //
6294 // Idx1 = shuffle Idx1, Idx2
6295 // Idx1 = shuffle Idx1, Idx3
6296 SDValue &V = Values.try_emplace(Key: Idx1).first->getSecond();
6297 if (!V)
6298 V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
6299 (Idx1 % NumOfSrcRegs) * NumOpElts);
6300 if (Idx2 != UINT_MAX) {
6301 SDValue &V = Values.try_emplace(Key: Idx2).first->getSecond();
6302 if (!V)
6303 V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
6304 (Idx2 % NumOfSrcRegs) * NumOpElts);
6305 }
6306 }
6307 SDValue V;
6308 for (const auto &[Idx1, Idx2, Mask] : Data) {
6309 SDValue V1 = Values.at(Val: Idx1);
6310 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Val: Idx2);
6311 V = PerformShuffle(V1, V2, Mask);
6312 Values[Idx1] = V;
6313 }
6314
6315 unsigned InsertIdx = I * NumOpElts;
6316 V = convertToScalableVector(VT: M1VT, V, DAG, Subtarget);
6317 Vec = DAG.getInsertSubvector(DL, Vec, SubVec: V, Idx: InsertIdx);
6318 }
6319 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
6320}
6321
6322// Matches a subset of compress masks with a contiguous prefix of output
6323// elements. This could be extended to allow gaps by deciding which
6324// source elements to spuriously demand.
6325static bool isCompressMask(ArrayRef<int> Mask) {
6326 int Last = -1;
6327 bool SawUndef = false;
6328 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
6329 if (M == -1) {
6330 SawUndef = true;
6331 continue;
6332 }
6333 if (SawUndef)
6334 return false;
6335 if (Idx > (unsigned)M)
6336 return false;
6337 if (M <= Last)
6338 return false;
6339 Last = M;
6340 }
6341 return true;
6342}
6343
6344/// Given a shuffle where the indices are disjoint between the two sources,
6345/// e.g.:
6346///
6347/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
6348///
6349/// Merge the two sources into one and do a single source shuffle:
6350///
6351/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
6352/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
6353///
6354/// A vselect will either be merged into a masked instruction or be lowered as a
6355/// vmerge.vvm, which is cheaper than a vrgather.vv.
6356static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
6357 SelectionDAG &DAG,
6358 const RISCVSubtarget &Subtarget) {
6359 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6360 MVT XLenVT = Subtarget.getXLenVT();
6361 SDLoc DL(SVN);
6362
6363 const ArrayRef<int> Mask = SVN->getMask();
6364
6365 // Work out which source each lane will come from.
6366 SmallVector<int, 16> Srcs(Mask.size(), -1);
6367
6368 for (int Idx : Mask) {
6369 if (Idx == -1)
6370 continue;
6371 unsigned SrcIdx = Idx % Mask.size();
6372 int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
6373 if (Srcs[SrcIdx] == -1)
6374 // Mark this source as using this lane.
6375 Srcs[SrcIdx] = Src;
6376 else if (Srcs[SrcIdx] != Src)
6377 // The other source is using this lane: not disjoint.
6378 return SDValue();
6379 }
6380
6381 SmallVector<SDValue> SelectMaskVals;
6382 for (int Lane : Srcs) {
6383 if (Lane == -1)
6384 SelectMaskVals.push_back(Elt: DAG.getUNDEF(VT: XLenVT));
6385 else
6386 SelectMaskVals.push_back(Elt: DAG.getConstant(Val: Lane ? 0 : 1, DL, VT: XLenVT));
6387 }
6388 MVT MaskVT = VT.changeVectorElementType(EltVT: MVT::i1);
6389 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: SelectMaskVals);
6390 SDValue Select = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask,
6391 N2: SVN->getOperand(Num: 0), N3: SVN->getOperand(Num: 1));
6392
6393 // Move all indices relative to the first source.
6394 SmallVector<int> NewMask(Mask.size());
6395 for (unsigned I = 0; I < Mask.size(); I++) {
6396 if (Mask[I] == -1)
6397 NewMask[I] = -1;
6398 else
6399 NewMask[I] = Mask[I] % Mask.size();
6400 }
6401
6402 return DAG.getVectorShuffle(VT, dl: DL, N1: Select, N2: DAG.getUNDEF(VT), Mask: NewMask);
6403}
6404
6405/// Is this mask local (i.e. elements only move within their local span), and
6406/// repeating (that is, the same rearrangement is being done within each span)?
6407static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
6408 // Require a prefix from the original mask until the consumer code
6409 // is adjusted to rewrite the mask instead of just taking a prefix.
6410 for (auto [I, M] : enumerate(First&: Mask)) {
6411 if (M == -1)
6412 continue;
6413 if ((M / Span) != (int)(I / Span))
6414 return false;
6415 int SpanIdx = I % Span;
6416 int Expected = M % Span;
6417 if (Mask[SpanIdx] != Expected)
6418 return false;
6419 }
6420 return true;
6421}
6422
6423/// Is this mask only using elements from the first span of the input?
6424static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
6425 return all_of(Range&: Mask, P: [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
6426}
6427
6428/// Return true for a mask which performs an arbitrary shuffle within the first
6429/// span, and then repeats that same result across all remaining spans. Note
6430/// that this doesn't check if all the inputs come from a single span!
6431static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
6432 // Require a prefix from the original mask until the consumer code
6433 // is adjusted to rewrite the mask instead of just taking a prefix.
6434 for (auto [I, M] : enumerate(First&: Mask)) {
6435 if (M == -1)
6436 continue;
6437 int SpanIdx = I % Span;
6438 if (Mask[SpanIdx] != M)
6439 return false;
6440 }
6441 return true;
6442}
6443
6444/// Try to widen element type to get a new mask value for a better permutation
6445/// sequence. This doesn't try to inspect the widened mask for profitability;
6446/// we speculate the widened form is equal or better. This has the effect of
6447/// reducing mask constant sizes - allowing cheaper materialization sequences
6448/// - and index sequence sizes - reducing register pressure and materialization
6449/// cost, at the cost of (possibly) an extra VTYPE toggle.
6450static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
6451 SDLoc DL(Op);
6452 MVT VT = Op.getSimpleValueType();
6453 MVT ScalarVT = VT.getVectorElementType();
6454 unsigned ElementSize = ScalarVT.getFixedSizeInBits();
6455 SDValue V0 = Op.getOperand(i: 0);
6456 SDValue V1 = Op.getOperand(i: 1);
6457 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
6458
6459 // Avoid wasted work leading to isTypeLegal check failing below
6460 if (ElementSize > 32)
6461 return SDValue();
6462
6463 SmallVector<int, 8> NewMask;
6464 if (!widenShuffleMaskElts(M: Mask, NewMask))
6465 return SDValue();
6466
6467 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: ElementSize * 2)
6468 : MVT::getIntegerVT(BitWidth: ElementSize * 2);
6469 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
6470 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT))
6471 return SDValue();
6472 V0 = DAG.getBitcast(VT: NewVT, V: V0);
6473 V1 = DAG.getBitcast(VT: NewVT, V: V1);
6474 return DAG.getBitcast(VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: V0, N2: V1, Mask: NewMask));
6475}
6476
6477// Match an interleave shuffle that forms a P-extension packed zip:
6478// <a0, b0, a1, b1, ...> -> zip*p/wzip*p
6479static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
6480 const RISCVSubtarget &Subtarget,
6481 SelectionDAG &DAG) {
6482 SDValue V1 = SVN->getOperand(Num: 0);
6483 SDValue V2 = SVN->getOperand(Num: 1);
6484 SDLoc DL(SVN);
6485 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6486 unsigned NumElts = VT.getVectorNumElements();
6487 ArrayRef<int> Mask = SVN->getMask();
6488
6489 if (VT != MVT::v8i8 && VT != MVT::v4i16)
6490 return SDValue();
6491
6492 SmallVector<unsigned, 2> StartIndexes;
6493 if (!V2.isUndef() &&
6494 ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: NumElts * 2, StartIndexes)) {
6495 unsigned EvenSrc = StartIndexes[0];
6496 unsigned OddSrc = StartIndexes[1];
6497 if (EvenSrc == 0 && OddSrc == NumElts) {
6498 if (Subtarget.is64Bit())
6499 return DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT, N1: V1, N2: V2);
6500 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6501 V1 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V1, Idx: 0);
6502 V2 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V2, Idx: 0);
6503 return DAG.getNode(Opcode: RISCVISD::PWZIP, DL, VT, N1: V1, N2: V2);
6504 }
6505 if (EvenSrc == NumElts && OddSrc == 0) {
6506 if (Subtarget.is64Bit())
6507 return DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT, N1: V2, N2: V1);
6508 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6509 V1 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V1, Idx: 0);
6510 V2 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V2, Idx: 0);
6511 return DAG.getNode(Opcode: RISCVISD::PWZIP, DL, VT, N1: V2, N2: V1);
6512 }
6513 }
6514
6515 return SDValue();
6516}
6517
6518// Match a deinterleave shuffle that forms a P-extension packed unzip:
6519// <a0, a2, ..., b0, b2, ...> -> unzip*p
6520// <a1, a3, ..., b1, b3, ...> -> unzip*hp
6521static SDValue lowerVECTOR_SHUFFLEAsPUnzip(ShuffleVectorSDNode *SVN,
6522 SelectionDAG &DAG, bool IsRV64) {
6523 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6524 if (!IsRV64 || (VT != MVT::v8i8 && VT != MVT::v4i16))
6525 return SDValue();
6526
6527 SDValue V1 = SVN->getOperand(Num: 0);
6528 SDValue V2 = SVN->getOperand(Num: 1);
6529 SDLoc DL(SVN);
6530 ArrayRef<int> Mask = SVN->getMask();
6531
6532 unsigned Index = 0;
6533 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index))
6534 return SDValue();
6535
6536 unsigned Opc = Index == 0 ? RISCVISD::PUNZIPE : RISCVISD::PUNZIPO;
6537 return DAG.getNode(Opcode: Opc, DL, VT, N1: V1, N2: V2);
6538}
6539
6540// Match the packed zero-extend shuffle mask <0, N, 2, N+2, ...>: even result
6541// lanes keep operand 0's even lanes and odd result lanes come from operand 1.
6542// The odd lanes may select any element of operand 1, which is looser than a
6543// strict pair-even mask; DAGCombiner::XformToShuffleWithZero forms exactly this
6544// from a packed zero-extend `and`, leaving each zeroed lane at its own
6545// position, and lowerVECTOR_SHUFFLEAsPPair then forms the PPAIRE. Undef lanes
6546// always match.
6547static bool isPackedZExtShuffleMask(ArrayRef<int> Mask) {
6548 unsigned NumElts = Mask.size();
6549 if (NumElts % 2 != 0)
6550 return false;
6551 for (unsigned I = 0; I != NumElts / 2; ++I)
6552 if ((Mask[2 * I] >= 0 && Mask[2 * I] != (int)(2 * I)) ||
6553 (Mask[2 * I + 1] >= 0 && Mask[2 * I + 1] < (int)NumElts))
6554 return false;
6555 return true;
6556}
6557
6558// Match a legalized deinterleave shuffle on two RV32 vector halves and lower
6559// it to an RV32 P narrowing shift on the concatenated source.
6560static SDValue
6561lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(ShuffleVectorSDNode *SVN,
6562 const RISCVSubtarget &Subtarget,
6563 SelectionDAG &DAG) {
6564 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6565 if (Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
6566 return SDValue();
6567
6568 SDValue V1 = SVN->getOperand(Num: 0);
6569 SDValue V2 = SVN->getOperand(Num: 1);
6570 SDLoc DL(SVN);
6571 unsigned NumElts = VT.getVectorNumElements();
6572
6573 SDValue Src = foldConcatVector(V1, V2);
6574 if (!Src) {
6575 MVT SrcVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
6576 Src = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: SrcVT, N1: V1, N2: V2);
6577 }
6578
6579 // The source vector should be twice the size.
6580 if (Src.getValueType().getVectorNumElements() != 2 * NumElts)
6581 return SDValue();
6582
6583 unsigned Index = 0;
6584 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask: SVN->getMask(), Factor: 2, Index))
6585 return SDValue();
6586
6587 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
6588 return DAG.getNode(Opcode: RISCVISD::PNSRL, DL, VT, N1: Src,
6589 N2: DAG.getConstant(Val: Index * EltBits, DL, VT: MVT::i32));
6590}
6591
6592// Match a strided-interleave shuffle that forms a P-extension packed pair:
6593// <a0, b0, a2, b2, ...> -> ppaire.*
6594// <a0, b1, a2, b3, ...> -> ppaireo.*
6595// <a1, b0, a3, b2, ...> -> ppairoe.*
6596// <a1, b1, a3, b3, ...> -> ppairo.*
6597static SDValue lowerVECTOR_SHUFFLEAsPPair(ShuffleVectorSDNode *SVN,
6598 SelectionDAG &DAG) {
6599 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6600 if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v4i16)
6601 return SDValue();
6602
6603 SDValue V1 = SVN->getOperand(Num: 0);
6604 SDValue V2 = SVN->getOperand(Num: 1);
6605 SDLoc DL(SVN);
6606 unsigned NumElts = VT.getVectorNumElements();
6607 ArrayRef<int> Mask = SVN->getMask();
6608
6609 // A splat operand's lanes are all equal, so a lane selecting from it matches
6610 // any of its positions. This covers the zero operand XformToShuffleWithZero
6611 // forms for a packed zero-extend, which keeps each zeroed lane at its own
6612 // position rather than the strided one.
6613 bool V1IsSplat = DAG.isSplatValue(V: V1);
6614 bool V2IsSplat = DAG.isSplatValue(V: V2);
6615
6616 // Walk the mask once, tracking the operand feeding the destination's even
6617 // lanes (index 0) and the operand feeding its odd lanes (index 1) — either
6618 // may turn out to be V1 or V2 — along with whether each pulls the even or
6619 // odd element out of its pair. All even (resp. odd) lanes must agree on
6620 // both the operand and the parity used; a splat operand's lanes are all
6621 // equal so it never constrains the parity.
6622 SDValue Src[2];
6623 std::optional<bool> Parity[2];
6624 for (unsigned I = 0; I != NumElts; ++I) {
6625 int M = Mask[I];
6626 if (M < 0)
6627 continue;
6628 unsigned Lane = I % 2;
6629 bool FromV1 = (unsigned)M < NumElts;
6630 SDValue Cand = FromV1 ? V1 : V2;
6631 unsigned Local = (unsigned)M % NumElts;
6632 if (!Src[Lane])
6633 Src[Lane] = Cand;
6634 else if (Src[Lane] != Cand)
6635 return SDValue();
6636
6637 // Splats don't constrain parity.
6638 if (FromV1 ? V1IsSplat : V2IsSplat)
6639 continue;
6640
6641 // The index must be from the even/odd element of its pair.
6642 if (Local / 2 != I / 2)
6643 return SDValue();
6644
6645 bool P = Local % 2;
6646 if (!Parity[Lane])
6647 Parity[Lane] = P;
6648 else if (*Parity[Lane] != P)
6649 return SDValue();
6650 }
6651
6652 // Make sure we have a source for both lanes.
6653 if (!Src[0] || !Src[1])
6654 return SDValue();
6655
6656 bool EvenIsOdd = Parity[0].value_or(u: false);
6657 bool OddIsOdd = Parity[1].value_or(u: false);
6658 unsigned Opc;
6659 if (!EvenIsOdd && !OddIsOdd)
6660 Opc = RISCVISD::PPAIRE;
6661 else if (EvenIsOdd && OddIsOdd)
6662 Opc = RISCVISD::PPAIRO;
6663 else if (!EvenIsOdd && OddIsOdd)
6664 Opc = RISCVISD::PPAIREO;
6665 else
6666 Opc = RISCVISD::PPAIROE;
6667
6668 return DAG.getNode(Opcode: Opc, DL, VT, N1: Src[0], N2: Src[1]);
6669}
6670
6671SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6672 SelectionDAG &DAG) const {
6673 SDValue V1 = Op.getOperand(i: 0);
6674 SDValue V2 = Op.getOperand(i: 1);
6675 SDLoc DL(Op);
6676 MVT XLenVT = Subtarget.getXLenVT();
6677 MVT VT = Op.getSimpleValueType();
6678 unsigned NumElts = VT.getVectorNumElements();
6679 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6680
6681 // Select RVP-specific packed shuffles before falling back to the generic
6682 // fixed/scalable-vector lowering below.
6683 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
6684 ArrayRef<int> Mask = SVN->getMask();
6685
6686 // Select an element reverse shuffle to VECTOR_REVERSE. The tablegen
6687 // patterns select rev8/rev16/ppairoe.* from VECTOR_REVERSE.
6688 // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
6689 // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
6690 auto IsLowReverse = [&](unsigned L) {
6691 return V2.isUndef() &&
6692 ShuffleVectorInst::isReverseMask(Mask: Mask.take_front(N: L), NumSrcElts: L) &&
6693 all_of(Range: Mask.drop_front(N: L), P: [](int M) { return M < 0; });
6694 };
6695 if (IsLowReverse(NumElts))
6696 return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1);
6697 if (Subtarget.is64Bit() && VT == MVT::v4i16 && IsLowReverse(/*L=*/2))
6698 return DAG.getNode(Opcode: RISCVISD::PPAIROE_H, DL, VT, N1: V1, N2: V1);
6699 // Widened: reversing sends the low-half lanes to the top half, so shift
6700 // them back down by half the register. Only the 64-bit packed types are
6701 // legal here, so the register is XLen (i64).
6702 if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
6703 IsLowReverse(NumElts / 2)) {
6704 SDValue Rev = DAG.getBitcast(
6705 VT: MVT::i64, V: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1));
6706 SDValue Srl =
6707 DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Rev,
6708 N2: DAG.getConstant(Val: VT.getSizeInBits() / 2, DL, VT: MVT::i64));
6709 return DAG.getBitcast(VT, V: Srl);
6710 }
6711
6712 if (SDValue V = lowerVECTOR_SHUFFLEAsPUnzip(SVN, DAG, IsRV64: Subtarget.is64Bit()))
6713 return V;
6714 if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, Subtarget, DAG))
6715 return V;
6716 if (SDValue V =
6717 lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(SVN, Subtarget, DAG))
6718 return V;
6719 if (SDValue V = lowerVECTOR_SHUFFLEAsPPair(SVN, DAG))
6720 return V;
6721 return SDValue();
6722 }
6723
6724 if (VT.getVectorElementType() == MVT::i1) {
6725 // Lower to a vror.vi of a larger element type if possible before we promote
6726 // i1s to i8s.
6727 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6728 return V;
6729 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
6730 return V;
6731
6732 // Promote i1 shuffle to i8 shuffle.
6733 MVT WidenVT = MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount());
6734 V1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: V1);
6735 V2 = V2.isUndef() ? DAG.getUNDEF(VT: WidenVT)
6736 : DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: V2);
6737 SDValue Shuffled = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: V1, N2: V2, Mask: SVN->getMask());
6738 return DAG.getSetCC(DL, VT, LHS: Shuffled, RHS: DAG.getConstant(Val: 0, DL, VT: WidenVT),
6739 Cond: ISD::SETNE);
6740 }
6741
6742 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6743
6744 // Store the return value in a single variable instead of structured bindings
6745 // so that we can pass it to GetSlide below, which cannot capture structured
6746 // bindings until C++20.
6747 auto TrueMaskVL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
6748 auto [TrueMask, VL] = TrueMaskVL;
6749
6750 if (SVN->isSplat()) {
6751 const int Lane = SVN->getSplatIndex();
6752 if (Lane >= 0) {
6753 MVT SVT = VT.getVectorElementType();
6754
6755 // Turn splatted vector load into a strided load with an X0 stride.
6756 SDValue V = V1;
6757 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
6758 // with undef.
6759 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
6760 int Offset = Lane;
6761 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
6762 int OpElements =
6763 V.getOperand(i: 0).getSimpleValueType().getVectorNumElements();
6764 V = V.getOperand(i: Offset / OpElements);
6765 Offset %= OpElements;
6766 }
6767
6768 // We need to ensure the load isn't atomic or volatile.
6769 if (ISD::isNormalLoad(N: V.getNode()) && cast<LoadSDNode>(Val&: V)->isSimple()) {
6770 auto *Ld = cast<LoadSDNode>(Val&: V);
6771 Offset *= SVT.getStoreSize();
6772 SDValue NewAddr = DAG.getMemBasePlusOffset(
6773 Base: Ld->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: Offset), DL);
6774
6775 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
6776 if (SVT.isInteger() && SVT.bitsGT(VT: XLenVT)) {
6777 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
6778 SDValue IntID =
6779 DAG.getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: XLenVT);
6780 SDValue Ops[] = {Ld->getChain(),
6781 IntID,
6782 DAG.getUNDEF(VT: ContainerVT),
6783 NewAddr,
6784 DAG.getRegister(Reg: RISCV::X0, VT: XLenVT),
6785 VL};
6786 SDValue NewLoad = DAG.getMemIntrinsicNode(
6787 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT: SVT,
6788 MMO: DAG.getMachineFunction().getMachineMemOperand(
6789 MMO: Ld->getMemOperand(), Offset, Size: SVT.getStoreSize()));
6790 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: NewLoad);
6791 return convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
6792 }
6793
6794 MVT SplatVT = ContainerVT;
6795
6796 // f16 with zvfhmin and bf16 need to use an integer scalar load.
6797 if (SVT == MVT::bf16 ||
6798 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
6799 SVT = MVT::i16;
6800 SplatVT = ContainerVT.changeVectorElementType(EltVT: SVT);
6801 }
6802
6803 // Otherwise use a scalar load and splat. This will give the best
6804 // opportunity to fold a splat into the operation. ISel can turn it into
6805 // the x0 strided load if we aren't able to fold away the select.
6806 if (SVT.isFloatingPoint())
6807 V = DAG.getLoad(VT: SVT, dl: DL, Chain: Ld->getChain(), Ptr: NewAddr,
6808 PtrInfo: Ld->getPointerInfo().getWithOffset(O: Offset),
6809 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
6810 else
6811 V = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: XLenVT, Chain: Ld->getChain(), Ptr: NewAddr,
6812 PtrInfo: Ld->getPointerInfo().getWithOffset(O: Offset), MemVT: SVT,
6813 Alignment: Ld->getBaseAlign(),
6814 MMOFlags: Ld->getMemOperand()->getFlags());
6815 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: V);
6816
6817 unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
6818 : RISCVISD::VMV_V_X_VL;
6819 SDValue Splat =
6820 DAG.getNode(Opcode: Opc, DL, VT: SplatVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: V, N3: VL);
6821 Splat = DAG.getBitcast(VT: ContainerVT, V: Splat);
6822 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
6823 }
6824
6825 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
6826 assert(Lane < (int)NumElts && "Unexpected lane!");
6827 SDValue Gather = DAG.getNode(Opcode: RISCVISD::VRGATHER_VX_VL, DL, VT: ContainerVT,
6828 N1: V1, N2: DAG.getConstant(Val: Lane, DL, VT: XLenVT),
6829 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
6830 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6831 }
6832 }
6833
6834 // For exact VLEN m2 or greater, try to split to m1 operations if we
6835 // can split cleanly.
6836 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
6837 return V;
6838
6839 ArrayRef<int> Mask = SVN->getMask();
6840
6841 if (SDValue V =
6842 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
6843 return V;
6844
6845 if (SDValue V =
6846 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
6847 return V;
6848
6849 // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
6850 // available.
6851 if (Subtarget.hasStdExtZvkb())
6852 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6853 return V;
6854
6855 if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: NumElts) && V2.isUndef() &&
6856 NumElts != 2)
6857 return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1);
6858
6859 // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
6860 // use shift and truncate to perform the shuffle.
6861 // TODO: For Factor=6, we can perform the first step of the deinterleave via
6862 // shift-and-trunc reducing total cost for everything except an mf8 result.
6863 // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
6864 // to do the entire operation.
6865 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6866 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6867 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6868 for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
6869 unsigned Index = 0;
6870 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
6871 1 < count_if(Range&: Mask, P: [](int Idx) { return Idx != -1; })) {
6872 if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
6873 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
6874 if (1 < count_if(Range&: Mask,
6875 P: [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6876 1 < count_if(Range&: Mask, P: [&Mask](int Idx) {
6877 return Idx >= (int)Mask.size();
6878 })) {
6879 // Narrow each source and concatenate them.
6880 // FIXME: For small LMUL it is better to concatenate first.
6881 MVT EltVT = VT.getVectorElementType();
6882 auto EltCnt = VT.getVectorElementCount();
6883 MVT SubVT =
6884 MVT::getVectorVT(VT: EltVT, EC: EltCnt.divideCoefficientBy(RHS: Factor));
6885
6886 SDValue Lo =
6887 getDeinterleaveShiftAndTrunc(DL, VT: SubVT, Src: V1, Factor, Index, DAG);
6888 SDValue Hi =
6889 getDeinterleaveShiftAndTrunc(DL, VT: SubVT, Src: V2, Factor, Index, DAG);
6890
6891 SDValue Concat =
6892 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL,
6893 VT: SubVT.getDoubleNumVectorElementsVT(), N1: Lo, N2: Hi);
6894 if (Factor == 2)
6895 return Concat;
6896
6897 SDValue Vec = DAG.getUNDEF(VT);
6898 return DAG.getInsertSubvector(DL, Vec, SubVec: Concat, Idx: 0);
6899 }
6900 }
6901 }
6902 }
6903
6904 // If this is a deinterleave(2), try using vunzip{e,o}. This mostly catches
6905 // e64 which can't match above.
6906 unsigned Index = 0;
6907 if (Subtarget.hasStdExtZvzip() &&
6908 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index) &&
6909 1 < count_if(Range&: Mask, P: [](int Idx) { return Idx != -1; })) {
6910 bool UsesBothSources =
6911 1 < count_if(Range&: Mask,
6912 P: [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6913 1 < count_if(Range&: Mask,
6914 P: [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
6915
6916 unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
6917 if (isLegalVTForZvzipDeinterleavedOperand(VT, Subtarget)) {
6918 MVT NewVT = VT.getDoubleNumVectorElementsVT();
6919 if (isTypeLegal(VT: NewVT)) {
6920 SDValue Op;
6921 if (V2.isUndef()) {
6922 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, N1: V1, N2: V2);
6923 } else if (auto VLEN = Subtarget.getRealVLen();
6924 VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
6925 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, N1: V1, N2: V2);
6926 } else if (SDValue Src = foldConcatVector(V1, V2)) {
6927 Op = DAG.getExtractSubvector(DL, VT: NewVT, Vec: Src, Idx: 0);
6928 }
6929 if (Op)
6930 return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
6931 }
6932 }
6933
6934 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6935 if (UsesBothSources &&
6936 isLegalVTForZvzipDeinterleavedOperand(VT: HalfVT, Subtarget) &&
6937 V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
6938 V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
6939 SDValue Lo = lowerZvzipVUNZIP(Opc, Op: V1, DL, DAG, Subtarget);
6940 SDValue Hi = lowerZvzipVUNZIP(Opc, Op: V2, DL, DAG, Subtarget);
6941 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
6942 }
6943 }
6944
6945 if (SDValue V =
6946 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
6947 return V;
6948
6949 // Detect an interleave shuffle and lower to
6950 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
6951 int EvenSrc, OddSrc;
6952 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
6953 !(NumElts == 2 &&
6954 ShuffleVectorInst::isSingleSourceMask(Mask, NumSrcElts: Mask.size()))) {
6955 // Extract the halves of the vectors.
6956 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6957
6958 // Recognize if one half is actually undef; the matching above will
6959 // otherwise reuse the even stream for the undef one. This improves
6960 // spread(2) shuffles.
6961 bool LaneIsUndef[2] = { true, true};
6962 for (const auto &[Idx, M] : enumerate(First&: Mask))
6963 LaneIsUndef[Idx % 2] &= (M == -1);
6964
6965 int Size = Mask.size();
6966 SDValue EvenV, OddV;
6967 if (LaneIsUndef[0]) {
6968 EvenV = DAG.getUNDEF(VT: HalfVT);
6969 } else {
6970 assert(EvenSrc >= 0 && "Undef source?");
6971 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
6972 EvenV = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: EvenV, Idx: EvenSrc % Size);
6973 }
6974
6975 if (LaneIsUndef[1]) {
6976 OddV = DAG.getUNDEF(VT: HalfVT);
6977 } else {
6978 assert(OddSrc >= 0 && "Undef source?");
6979 OddV = (OddSrc / Size) == 0 ? V1 : V2;
6980 OddV = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: OddV, Idx: OddSrc % Size);
6981 }
6982
6983 // Prefer vzip if available.
6984 // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
6985 if (Subtarget.hasStdExtZvzip() &&
6986 isLegalVTForZvzipInterleavedOperand(VT, Subtarget))
6987 return lowerZvzipVZIP(Op0: EvenV, Op1: OddV, DL, DAG, Subtarget);
6988 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
6989 }
6990
6991 // Recognize a pattern which can handled via a pair of vslideup/vslidedown
6992 // instructions (in any combination) with masking on the second instruction.
6993 // Also handles masked slides into an identity source, and single slides
6994 // without masking. Avoid matching bit rotates (which are not also element
6995 // rotates) as slide pairs. This is a performance heuristic, not a
6996 // functional check.
6997 std::array<std::pair<int, int>, 2> SrcInfo;
6998 unsigned RotateAmt;
6999 MVT RotateVT;
7000 if (::isMaskedSlidePair(Mask, SrcInfo) &&
7001 (isElementRotate(SrcInfo, NumElts) ||
7002 !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
7003 SDValue Sources[2];
7004 auto GetSourceFor = [&](const std::pair<int, int> &Info) {
7005 int SrcIdx = Info.first;
7006 assert(SrcIdx == 0 || SrcIdx == 1);
7007 SDValue &Src = Sources[SrcIdx];
7008 if (!Src) {
7009 SDValue SrcV = SrcIdx == 0 ? V1 : V2;
7010 Src = convertToScalableVector(VT: ContainerVT, V: SrcV, DAG, Subtarget);
7011 }
7012 return Src;
7013 };
7014 auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
7015 SDValue Passthru) {
7016 auto [TrueMask, VL] = TrueMaskVL;
7017 SDValue SrcV = GetSourceFor(Src);
7018 int SlideAmt = Src.second;
7019 if (SlideAmt == 0) {
7020 // Should never be second operation
7021 assert(Mask == TrueMask);
7022 return SrcV;
7023 }
7024 if (SlideAmt < 0)
7025 return getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: SrcV,
7026 Offset: DAG.getConstant(Val: -SlideAmt, DL, VT: XLenVT), Mask, VL,
7027 Policy: RISCVVType::TAIL_AGNOSTIC);
7028 return getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: SrcV,
7029 Offset: DAG.getConstant(Val: SlideAmt, DL, VT: XLenVT), Mask, VL,
7030 Policy: RISCVVType::TAIL_AGNOSTIC);
7031 };
7032
7033 if (SrcInfo[1].first == -1) {
7034 SDValue Res = DAG.getUNDEF(VT: ContainerVT);
7035 Res = GetSlide(SrcInfo[0], TrueMask, Res);
7036 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
7037 }
7038
7039 if (Subtarget.hasStdExtZvzip()) {
7040 bool TryWiden = false;
7041 unsigned Factor;
7042 if (isPairEven(SrcInfo, Mask, Factor)) {
7043 if (Factor == 1) {
7044 SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
7045 SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
7046 return lowerZvzipVPAIR(Opc: RISCVISD::VPAIRE_VL, Op0: Src1, Op1: Src2, DL, DAG,
7047 Subtarget);
7048 }
7049 TryWiden = true;
7050 }
7051 if (isPairOdd(SrcInfo, Mask, Factor)) {
7052 if (Factor == 1) {
7053 SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
7054 SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
7055 return lowerZvzipVPAIR(Opc: RISCVISD::VPAIRO_VL, Op0: Src1, Op1: Src2, DL, DAG,
7056 Subtarget);
7057 }
7058 TryWiden = true;
7059 }
7060 // If we found a widening oppurtunity which would let us form a
7061 // pair-even or pair-odd, use the generic code to widen the shuffle
7062 // and recurse through this logic.
7063 if (TryWiden)
7064 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7065 return V;
7066 }
7067
7068 // Build the mask. Note that vslideup unconditionally preserves elements
7069 // below the slide amount in the destination, and thus those elements are
7070 // undefined in the mask. If the mask ends up all true (or undef), it
7071 // will be folded away by general logic.
7072 SmallVector<SDValue> MaskVals;
7073 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
7074 if (M < 0 ||
7075 (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
7076 MaskVals.push_back(Elt: DAG.getUNDEF(VT: XLenVT));
7077 continue;
7078 }
7079 int Src = M >= (int)NumElts;
7080 int Diff = (int)Idx - (M % NumElts);
7081 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
7082 assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
7083 "Must match exactly one of the two slides");
7084 MaskVals.push_back(Elt: DAG.getConstant(Val: C, DL, VT: XLenVT));
7085 }
7086 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7087 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
7088 SDValue SelectMask = convertToScalableVector(
7089 VT: ContainerVT.changeVectorElementType(EltVT: MVT::i1),
7090 V: DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals), DAG, Subtarget);
7091
7092 SDValue Res = DAG.getUNDEF(VT: ContainerVT);
7093 Res = GetSlide(SrcInfo[0], TrueMask, Res);
7094 Res = GetSlide(SrcInfo[1], SelectMask, Res);
7095 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
7096 }
7097
7098 // Handle any remaining single source shuffles
7099 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
7100 if (V2.isUndef()) {
7101 // We might be able to express the shuffle as a bitrotate. But even if we
7102 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
7103 // shifts and a vor will have a higher throughput than a vrgather.
7104 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
7105 return V;
7106
7107 if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
7108 return V;
7109
7110 // Match a spread(4,8) which can be done via extend and shift. Spread(2)
7111 // is fully covered in interleave(2) above, so it is ignored here.
7112 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
7113 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
7114 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
7115 for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
7116 unsigned Index;
7117 if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
7118 MVT NarrowVT =
7119 MVT::getVectorVT(VT: VT.getVectorElementType(), NumElements: NumElts / Factor);
7120 SDValue Src = DAG.getExtractSubvector(DL, VT: NarrowVT, Vec: V1, Idx: 0);
7121 return getWideningSpread(V: Src, Factor, Index, DL, DAG);
7122 }
7123 }
7124 }
7125
7126 // If only a prefix of the source elements influence a prefix of the
7127 // destination elements, try to see if we can reduce the required LMUL
7128 unsigned MinVLen = Subtarget.getRealMinVLen();
7129 unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
7130 if (NumElts > MinVLMAX) {
7131 unsigned MaxIdx = 0;
7132 for (auto [I, M] : enumerate(First&: Mask)) {
7133 if (M == -1)
7134 continue;
7135 MaxIdx = std::max(l: {(unsigned)I, (unsigned)M, MaxIdx});
7136 }
7137 unsigned NewNumElts =
7138 std::max(a: (uint64_t)MinVLMAX, b: PowerOf2Ceil(A: MaxIdx + 1));
7139 if (NewNumElts != NumElts) {
7140 MVT NewVT = MVT::getVectorVT(VT: VT.getVectorElementType(), NumElements: NewNumElts);
7141 V1 = DAG.getExtractSubvector(DL, VT: NewVT, Vec: V1, Idx: 0);
7142 SDValue Res = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT: NewVT),
7143 Mask: Mask.take_front(N: NewNumElts));
7144 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: Res, Idx: 0);
7145 }
7146 }
7147
7148 // Before hitting generic lowering fallbacks, try to widen the mask
7149 // to a wider SEW.
7150 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7151 return V;
7152
7153 // Can we generate a vcompress instead of a vrgather? These scale better
7154 // at high LMUL, at the cost of not being able to fold a following select
7155 // into them. The mask constants are also smaller than the index vector
7156 // constants, and thus easier to materialize.
7157 if (isCompressMask(Mask)) {
7158 SmallVector<SDValue> MaskVals(NumElts,
7159 DAG.getConstant(Val: false, DL, VT: XLenVT));
7160 for (auto Idx : Mask) {
7161 if (Idx == -1)
7162 break;
7163 assert(Idx >= 0 && (unsigned)Idx < NumElts);
7164 MaskVals[Idx] = DAG.getConstant(Val: true, DL, VT: XLenVT);
7165 }
7166 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
7167 SDValue CompressMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
7168 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT, N1: V1, N2: CompressMask,
7169 N3: DAG.getUNDEF(VT));
7170 }
7171
7172 if (VT.getScalarSizeInBits() == 8 &&
7173 any_of(Range&: Mask, P: [&](const auto &Idx) { return Idx > 255; })) {
7174 // On such a vector we're unable to use i8 as the index type.
7175 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
7176 // may involve vector splitting if we're already at LMUL=8, or our
7177 // user-supplied maximum fixed-length LMUL.
7178 return SDValue();
7179 }
7180
7181 // Base case for the two operand recursion below - handle the worst case
7182 // single source shuffle.
7183 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
7184 MVT IndexVT = VT.changeTypeToInteger();
7185 // Since we can't introduce illegal index types at this stage, use i16 and
7186 // vrgatherei16 if the corresponding index type for plain vrgather is greater
7187 // than XLenVT.
7188 if (IndexVT.getScalarType().bitsGT(VT: XLenVT)) {
7189 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7190 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
7191 }
7192
7193 // If the mask allows, we can do all the index computation in 16 bits. This
7194 // requires less work and less register pressure at high LMUL, and creates
7195 // smaller constants which may be cheaper to materialize.
7196 if (IndexVT.getScalarType().bitsGT(VT: MVT::i16) && isUInt<16>(x: NumElts - 1) &&
7197 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
7198 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7199 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
7200 }
7201
7202 MVT IndexContainerVT =
7203 ContainerVT.changeVectorElementType(EltVT: IndexVT.getScalarType());
7204
7205 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
7206 SmallVector<SDValue> GatherIndicesLHS;
7207 for (int MaskIndex : Mask) {
7208 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
7209 GatherIndicesLHS.push_back(Elt: IsLHSIndex
7210 ? DAG.getConstant(Val: MaskIndex, DL, VT: XLenVT)
7211 : DAG.getUNDEF(VT: XLenVT));
7212 }
7213 SDValue LHSIndices = DAG.getBuildVector(VT: IndexVT, DL, Ops: GatherIndicesLHS);
7214 LHSIndices =
7215 convertToScalableVector(VT: IndexContainerVT, V: LHSIndices, DAG, Subtarget);
7216 // At m1 and less, there's no point trying any of the high LMUL splitting
7217 // techniques. TODO: Should we reconsider this for DLEN < VLEN?
7218 if (NumElts <= MinVLMAX) {
7219 SDValue Gather = DAG.getNode(Opcode: GatherVVOpc, DL, VT: ContainerVT, N1: V1, N2: LHSIndices,
7220 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
7221 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7222 }
7223
7224 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
7225 EVT SubIndexVT = M1VT.changeVectorElementType(EltVT: IndexVT.getScalarType());
7226 auto [InnerTrueMask, InnerVL] =
7227 getDefaultScalableVLOps(VecVT: M1VT, DL, DAG, Subtarget);
7228 int N =
7229 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
7230 assert(isPowerOf2_32(N) && N <= 8);
7231
7232 // If we have a locally repeating mask, then we can reuse the first
7233 // register in the index register group for all registers within the
7234 // source register group. TODO: This generalizes to m2, and m4.
7235 if (isLocalRepeatingShuffle(Mask, Span: MinVLMAX)) {
7236 SDValue SubIndex = DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
7237 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
7238 for (int i = 0; i < N; i++) {
7239 unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
7240 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: SubIdx);
7241 SDValue SubVec =
7242 DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
7243 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
7244 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec, Idx: SubIdx);
7245 }
7246 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7247 }
7248
7249 // If we have a shuffle which only uses the first register in our source
7250 // register group, and repeats the same index across all spans, we can
7251 // use a single vrgather (and possibly some register moves).
7252 // TODO: This can be generalized for m2 or m4, or for any shuffle for
7253 // which we can do a linear number of shuffles to form an m1 which
7254 // contains all the output elements.
7255 if (isLowSourceShuffle(Mask, Span: MinVLMAX) &&
7256 isSpanSplatShuffle(Mask, Span: MinVLMAX)) {
7257 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: 0);
7258 SDValue SubIndex = DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
7259 SDValue SubVec = DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
7260 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
7261 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
7262 for (int i = 0; i < N; i++)
7263 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec,
7264 Idx: M1VT.getVectorMinNumElements() * i);
7265 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7266 }
7267
7268 // If we have a shuffle which only uses the first register in our
7269 // source register group, we can do a linear number of m1 vrgathers
7270 // reusing the same source register (but with different indices)
7271 // TODO: This can be generalized for m2 or m4, or for any shuffle
7272 // for which we can do a vslidedown followed by this expansion.
7273 if (isLowSourceShuffle(Mask, Span: MinVLMAX)) {
7274 SDValue SlideAmt =
7275 DAG.getElementCount(DL, VT: XLenVT, EC: M1VT.getVectorElementCount());
7276 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: 0);
7277 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
7278 for (int i = 0; i < N; i++) {
7279 if (i != 0)
7280 LHSIndices = getVSlidedown(DAG, Subtarget, DL, VT: IndexContainerVT,
7281 Passthru: DAG.getUNDEF(VT: IndexContainerVT), Op: LHSIndices,
7282 Offset: SlideAmt, Mask: TrueMask, VL);
7283 SDValue SubIndex =
7284 DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
7285 SDValue SubVec =
7286 DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
7287 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
7288 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec,
7289 Idx: M1VT.getVectorMinNumElements() * i);
7290 }
7291 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7292 }
7293
7294 // Fallback to generic vrgather if we can't find anything better.
7295 // On many machines, this will be O(LMUL^2)
7296 SDValue Gather = DAG.getNode(Opcode: GatherVVOpc, DL, VT: ContainerVT, N1: V1, N2: LHSIndices,
7297 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
7298 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7299 }
7300
7301 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
7302 // merged with a second vrgather.
7303 SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
7304
7305 // Now construct the mask that will be used by the blended vrgather operation.
7306 // Construct the appropriate indices into each vector.
7307 for (int MaskIndex : Mask) {
7308 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
7309 ShuffleMaskLHS.push_back(Elt: IsLHSOrUndefIndex && MaskIndex >= 0
7310 ? MaskIndex : -1);
7311 ShuffleMaskRHS.push_back(Elt: IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
7312 }
7313
7314 // If the mask indices are disjoint between the two sources, we can lower it
7315 // as a vselect + a single source vrgather.vv. Don't do this if we think the
7316 // operands may end up being lowered to something cheaper than a vrgather.vv.
7317 if (!DAG.isSplatValue(V: V2) && !DAG.isSplatValue(V: V1) &&
7318 !ShuffleVectorSDNode::isSplatMask(Mask: ShuffleMaskLHS) &&
7319 !ShuffleVectorSDNode::isSplatMask(Mask: ShuffleMaskRHS) &&
7320 !ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskLHS, NumSrcElts: NumElts) &&
7321 !ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskRHS, NumSrcElts: NumElts))
7322 if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
7323 return V;
7324
7325 // Before hitting generic lowering fallbacks, try to widen the mask
7326 // to a wider SEW.
7327 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7328 return V;
7329
7330 // Try to pick a profitable operand order.
7331 bool SwapOps = DAG.isSplatValue(V: V2) && !DAG.isSplatValue(V: V1);
7332 SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskRHS, NumSrcElts: NumElts);
7333
7334 // Recursively invoke lowering for each operand if we had two
7335 // independent single source shuffles, and then combine the result via a
7336 // vselect. Note that the vselect will likely be folded back into the
7337 // second permute (vrgather, or other) by the post-isel combine.
7338 V1 = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT), Mask: ShuffleMaskLHS);
7339 V2 = DAG.getVectorShuffle(VT, dl: DL, N1: V2, N2: DAG.getUNDEF(VT), Mask: ShuffleMaskRHS);
7340
7341 SmallVector<SDValue> MaskVals;
7342 for (int MaskIndex : Mask) {
7343 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
7344 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
7345 }
7346
7347 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7348 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
7349 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
7350
7351 if (SwapOps)
7352 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: V1, N3: V2);
7353 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: V2, N3: V1);
7354}
7355
7356bool RISCVTargetLowering::isVectorClearMaskLegal(ArrayRef<int> M,
7357 EVT VT) const {
7358 // Enable DAGCombiner::XformToShuffleWithZero to rewrite a packed zero-extend
7359 // `and` into shuffle(src, zero, ...), which lowerVECTOR_SHUFFLE turns into
7360 // RISCVISD::PPAIRE. Accept the packed byte/halfword views that lowering
7361 // handles; the 32-bit views are illegal on RV64 but reachable before type
7362 // legalization widens them to the legal 64-bit view.
7363 if (!Subtarget.hasStdExtP() || !VT.isSimple())
7364 return false;
7365 MVT SVT = VT.getSimpleVT();
7366 return (SVT == MVT::v4i8 || SVT == MVT::v8i8 || SVT == MVT::v4i16) &&
7367 isPackedZExtShuffleMask(Mask: M);
7368}
7369
7370bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7371 // Only support legal VTs for other shuffles for now.
7372 if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
7373 return false;
7374
7375 // Support splats for any type. These should type legalize well.
7376 if (ShuffleVectorSDNode::isSplatMask(Mask: M))
7377 return true;
7378
7379 const unsigned NumElts = M.size();
7380 MVT SVT = VT.getSimpleVT();
7381
7382 // Not for i1 vectors.
7383 if (SVT.getScalarType() == MVT::i1)
7384 return false;
7385
7386 std::array<std::pair<int, int>, 2> SrcInfo;
7387 int Dummy1, Dummy2;
7388 return ShuffleVectorInst::isReverseMask(Mask: M, NumSrcElts: NumElts) ||
7389 (::isMaskedSlidePair(Mask: M, SrcInfo) &&
7390 isElementRotate(SrcInfo, NumElts)) ||
7391 isInterleaveShuffle(Mask: M, VT: SVT, EvenSrc&: Dummy1, OddSrc&: Dummy2, Subtarget);
7392}
7393
7394// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
7395// the exponent.
7396SDValue
7397RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
7398 SelectionDAG &DAG) const {
7399 MVT VT = Op.getSimpleValueType();
7400 unsigned EltSize = VT.getScalarSizeInBits();
7401 SDValue Src = Op.getOperand(i: 0);
7402 SDLoc DL(Op);
7403 MVT ContainerVT = VT;
7404
7405 // We choose FP type that can represent the value if possible. Otherwise, we
7406 // use rounding to zero conversion for correct exponent of the result.
7407 // TODO: Use f16 for i8 when possible?
7408 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
7409 if (!isTypeLegal(VT: MVT::getVectorVT(VT: FloatEltVT, EC: VT.getVectorElementCount())))
7410 FloatEltVT = MVT::f32;
7411 MVT FloatVT = MVT::getVectorVT(VT: FloatEltVT, EC: VT.getVectorElementCount());
7412
7413 // Legal types should have been checked in the RISCVTargetLowering
7414 // constructor.
7415 // TODO: Splitting may make sense in some cases.
7416 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
7417 "Expected legal float type!");
7418
7419 // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
7420 // The trailing zero count is equal to log2 of this single bit value.
7421 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
7422 SDValue Neg = DAG.getNegative(Val: Src, DL, VT);
7423 Src = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Src, N2: Neg);
7424 }
7425
7426 // We have a legal FP type, convert to it.
7427 SDValue FloatVal;
7428 if (FloatVT.bitsGT(VT)) {
7429 FloatVal = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVT, Operand: Src);
7430 } else {
7431 // Use RTZ to avoid rounding influencing exponent of FloatVal.
7432 if (VT.isFixedLengthVector()) {
7433 ContainerVT = getContainerForFixedLengthVector(VT);
7434 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
7435 }
7436 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
7437 SDValue RTZRM =
7438 DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: Subtarget.getXLenVT());
7439 MVT ContainerFloatVT =
7440 MVT::getVectorVT(VT: FloatEltVT, EC: ContainerVT.getVectorElementCount());
7441 FloatVal = DAG.getNode(Opcode: RISCVISD::VFCVT_RM_F_XU_VL, DL, VT: ContainerFloatVT,
7442 N1: Src, N2: Mask, N3: RTZRM, N4: VL);
7443 if (VT.isFixedLengthVector())
7444 FloatVal = convertFromScalableVector(VT: FloatVT, V: FloatVal, DAG, Subtarget);
7445 }
7446 // Bitcast to integer and shift the exponent to the LSB.
7447 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
7448 SDValue Bitcast = DAG.getBitcast(VT: IntVT, V: FloatVal);
7449 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
7450
7451 // Restore back to original type. Truncation after SRL is to generate vnsrl.
7452 SDValue Exp = DAG.getNode(Opcode: ISD::SRL, DL, VT: IntVT, N1: Bitcast,
7453 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: IntVT));
7454 if (IntVT.bitsLT(VT))
7455 Exp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Exp);
7456 else if (IntVT.bitsGT(VT))
7457 Exp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Exp);
7458
7459 // The exponent contains log2 of the value in biased form.
7460 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
7461 // For trailing zeros, we just need to subtract the bias.
7462 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
7463 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Exp,
7464 N2: DAG.getConstant(Val: ExponentBias, DL, VT));
7465
7466 // For leading zeros, we need to remove the bias and convert from log2 to
7467 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
7468 unsigned Adjust = ExponentBias + (EltSize - 1);
7469 SDValue Res =
7470 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: Adjust, DL, VT), N2: Exp);
7471
7472 // The above result with zero input equals to Adjust which is greater than
7473 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
7474 if (Op.getOpcode() == ISD::CTLZ)
7475 Res = DAG.getNode(Opcode: ISD::UMIN, DL, VT, N1: Res, N2: DAG.getConstant(Val: EltSize, DL, VT));
7476
7477 return Res;
7478}
7479
7480SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
7481 SelectionDAG &DAG) const {
7482 SDLoc DL(Op);
7483 MVT XLenVT = Subtarget.getXLenVT();
7484 SDValue Source = Op->getOperand(Num: 0);
7485 MVT SrcVT = Source.getSimpleValueType();
7486 SDValue Mask = Op->getOperand(Num: 1);
7487 SDValue EVL = Op->getOperand(Num: 2);
7488
7489 if (SrcVT.isFixedLengthVector()) {
7490 MVT ContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
7491 Source = convertToScalableVector(VT: ContainerVT, V: Source, DAG, Subtarget);
7492 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
7493 Subtarget);
7494 SrcVT = ContainerVT;
7495 }
7496
7497 // Convert to boolean vector.
7498 if (SrcVT.getScalarType() != MVT::i1) {
7499 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
7500 SrcVT = MVT::getVectorVT(VT: MVT::i1, EC: SrcVT.getVectorElementCount());
7501 Source = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: SrcVT,
7502 Ops: {Source, AllZero, DAG.getCondCode(Cond: ISD::SETNE),
7503 DAG.getUNDEF(VT: SrcVT), Mask, EVL});
7504 }
7505
7506 SDValue Res = DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Source, N2: Mask, N3: EVL);
7507 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
7508 // In this case, we can interpret poison as -1, so nothing to do further.
7509 return Res;
7510
7511 // Convert -1 to VL.
7512 SDValue SetCC =
7513 DAG.getSetCC(DL, VT: XLenVT, LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETLT);
7514 Res = DAG.getSelect(DL, VT: XLenVT, Cond: SetCC, LHS: EVL, RHS: Res);
7515 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: Res);
7516}
7517
7518// While RVV has alignment restrictions, we should always be able to load as a
7519// legal equivalently-sized byte-typed vector instead. This method is
7520// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
7521// the load is already correctly-aligned, it returns SDValue().
7522SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
7523 SelectionDAG &DAG) const {
7524 auto *Load = cast<LoadSDNode>(Val&: Op);
7525 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7526
7527 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7528 VT: Load->getMemoryVT(),
7529 MMO: *Load->getMemOperand()))
7530 return SDValue();
7531
7532 SDLoc DL(Op);
7533 MVT VT = Op.getSimpleValueType();
7534 unsigned EltSizeBits = VT.getScalarSizeInBits();
7535 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7536 "Unexpected unaligned RVV load type");
7537 MVT NewVT =
7538 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7539 assert(NewVT.isValid() &&
7540 "Expecting equally-sized RVV vector types to be legal");
7541 SDValue L = DAG.getLoad(VT: NewVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
7542 PtrInfo: Load->getPointerInfo(), Alignment: Load->getBaseAlign(),
7543 MMOFlags: Load->getMemOperand()->getFlags());
7544 return DAG.getMergeValues(Ops: {DAG.getBitcast(VT, V: L), L.getValue(R: 1)}, dl: DL);
7545}
7546
7547// While RVV has alignment restrictions, we should always be able to store as a
7548// legal equivalently-sized byte-typed vector instead. This method is
7549// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
7550// returns SDValue() if the store is already correctly aligned.
7551SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
7552 SelectionDAG &DAG) const {
7553 auto *Store = cast<StoreSDNode>(Val&: Op);
7554 assert(Store && Store->getValue().getValueType().isVector() &&
7555 "Expected vector store");
7556
7557 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7558 VT: Store->getMemoryVT(),
7559 MMO: *Store->getMemOperand()))
7560 return SDValue();
7561
7562 SDLoc DL(Op);
7563 SDValue StoredVal = Store->getValue();
7564 MVT VT = StoredVal.getSimpleValueType();
7565 unsigned EltSizeBits = VT.getScalarSizeInBits();
7566 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7567 "Unexpected unaligned RVV store type");
7568 MVT NewVT =
7569 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7570 assert(NewVT.isValid() &&
7571 "Expecting equally-sized RVV vector types to be legal");
7572 StoredVal = DAG.getBitcast(VT: NewVT, V: StoredVal);
7573 return DAG.getStore(Chain: Store->getChain(), dl: DL, Val: StoredVal, Ptr: Store->getBasePtr(),
7574 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
7575 MMOFlags: Store->getMemOperand()->getFlags());
7576}
7577
7578// While RVV has alignment restrictions, we should always be able to load as a
7579// legal equivalently-sized byte-typed vector instead. This method is
7580// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
7581// the load is already correctly-aligned, it returns SDValue().
7582SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
7583 SelectionDAG &DAG) const {
7584 auto *Load = cast<VPLoadSDNode>(Val&: Op);
7585 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7586
7587 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7588 VT: Load->getMemoryVT(),
7589 MMO: *Load->getMemOperand()))
7590 return SDValue();
7591
7592 SDValue Mask = Load->getMask();
7593
7594 // FIXME: Handled masked loads somehow.
7595 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
7596 return SDValue();
7597
7598 SDLoc DL(Op);
7599 MVT VT = Op.getSimpleValueType();
7600 unsigned EltSizeBits = VT.getScalarSizeInBits();
7601 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7602 "Unexpected unaligned RVV load type");
7603 MVT NewVT =
7604 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7605 assert(NewVT.isValid() &&
7606 "Expecting equally-sized RVV vector types to be legal");
7607
7608 SDValue VL = Load->getVectorLength();
7609 VL = DAG.getNode(Opcode: ISD::MUL, DL, VT: VL.getValueType(), N1: VL,
7610 N2: DAG.getConstant(Val: (EltSizeBits / 8), DL, VT: VL.getValueType()));
7611
7612 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, EC: NewVT.getVectorElementCount());
7613 SDValue L = DAG.getLoadVP(VT: NewVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
7614 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL,
7615 PtrInfo: Load->getPointerInfo(), Alignment: Load->getBaseAlign(),
7616 MMOFlags: Load->getMemOperand()->getFlags(), AAInfo: AAMDNodes());
7617 return DAG.getMergeValues(Ops: {DAG.getBitcast(VT, V: L), L.getValue(R: 1)}, dl: DL);
7618}
7619
7620// While RVV has alignment restrictions, we should always be able to store as a
7621// legal equivalently-sized byte-typed vector instead. This method is
7622// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
7623// It returns SDValue() if the store is already correctly aligned.
7624SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
7625 SelectionDAG &DAG) const {
7626 auto *Store = cast<VPStoreSDNode>(Val&: Op);
7627 assert(Store && Store->getValue().getValueType().isVector() &&
7628 "Expected vector store");
7629
7630 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7631 VT: Store->getMemoryVT(),
7632 MMO: *Store->getMemOperand()))
7633 return SDValue();
7634
7635 SDValue Mask = Store->getMask();
7636
7637 // FIXME: Handled masked stores somehow.
7638 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
7639 return SDValue();
7640
7641 SDLoc DL(Op);
7642 SDValue StoredVal = Store->getValue();
7643 MVT VT = StoredVal.getSimpleValueType();
7644 unsigned EltSizeBits = VT.getScalarSizeInBits();
7645 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7646 "Unexpected unaligned RVV store type");
7647 MVT NewVT =
7648 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7649 assert(NewVT.isValid() &&
7650 "Expecting equally-sized RVV vector types to be legal");
7651
7652 SDValue VL = Store->getVectorLength();
7653 VL = DAG.getNode(Opcode: ISD::MUL, DL, VT: VL.getValueType(), N1: VL,
7654 N2: DAG.getConstant(Val: (EltSizeBits / 8), DL, VT: VL.getValueType()));
7655
7656 StoredVal = DAG.getBitcast(VT: NewVT, V: StoredVal);
7657
7658 LocationSize Size = LocationSize::precise(Value: NewVT.getStoreSize());
7659 MachineFunction &MF = DAG.getMachineFunction();
7660 MachineMemOperand *MMO = MF.getMachineMemOperand(
7661 PtrInfo: Store->getPointerInfo(), F: Store->getMemOperand()->getFlags(), Size,
7662 BaseAlignment: Store->getBaseAlign());
7663
7664 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, EC: NewVT.getVectorElementCount());
7665 return DAG.getStoreVP(Chain: Store->getChain(), dl: DL, Val: StoredVal, Ptr: Store->getBasePtr(),
7666 Offset: DAG.getPOISON(VT: Store->getBasePtr().getValueType()),
7667 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL, MemVT: NewVT, MMO,
7668 AM: ISD::UNINDEXED);
7669}
7670
7671static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
7672 const RISCVSubtarget &Subtarget) {
7673 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
7674
7675 int64_t Imm = cast<ConstantSDNode>(Val&: Op)->getSExtValue();
7676
7677 // All simm32 constants should be handled by isel.
7678 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
7679 // this check redundant, but small immediates are common so this check
7680 // should have better compile time.
7681 if (isInt<32>(x: Imm))
7682 return Op;
7683
7684 // We only need to cost the immediate, if constant pool lowering is enabled.
7685 if (!Subtarget.useConstantPoolForLargeInts())
7686 return Op;
7687
7688 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
7689 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
7690 return Op;
7691
7692 // Optimizations below are disabled for opt size. If we're optimizing for
7693 // size, use a constant pool.
7694 if (DAG.shouldOptForSize())
7695 return SDValue();
7696
7697 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
7698 // that if it will avoid a constant pool.
7699 // It will require an extra temporary register though.
7700 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
7701 // low and high 32 bits are the same and bit 31 and 63 are set.
7702 unsigned ShiftAmt, AddOpc;
7703 RISCVMatInt::InstSeq SeqLo =
7704 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
7705 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
7706 return Op;
7707
7708 return SDValue();
7709}
7710
7711SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
7712 SelectionDAG &DAG) const {
7713 MVT VT = Op.getSimpleValueType();
7714 const APFloat &Imm = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7715
7716 // Can this constant be selected by a Zfa FLI instruction?
7717 bool Negate = false;
7718 int Index = getLegalZfaFPImm(Imm, VT);
7719
7720 // If the constant is negative, try negating.
7721 if (Index < 0 && Imm.isNegative()) {
7722 Index = getLegalZfaFPImm(Imm: -Imm, VT);
7723 Negate = true;
7724 }
7725
7726 // If we couldn't find a FLI lowering, fall back to generic code.
7727 if (Index < 0)
7728 return SDValue();
7729
7730 // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
7731 SDLoc DL(Op);
7732 SDValue Const =
7733 DAG.getNode(Opcode: RISCVISD::FLI, DL, VT,
7734 Operand: DAG.getTargetConstant(Val: Index, DL, VT: Subtarget.getXLenVT()));
7735 if (!Negate)
7736 return Const;
7737
7738 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Const);
7739}
7740
7741static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
7742 SelectionDAG &DAG) {
7743
7744 unsigned IsData = Op.getConstantOperandVal(i: 4);
7745
7746 // mips-p8700 we support data prefetch for now.
7747 if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
7748 return Op.getOperand(i: 0);
7749 return Op;
7750}
7751
7752static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
7753 const RISCVSubtarget &Subtarget) {
7754 SDLoc dl(Op);
7755 AtomicOrdering FenceOrdering =
7756 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
7757 SyncScope::ID FenceSSID =
7758 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
7759
7760 if (Subtarget.hasStdExtZtso()) {
7761 // The only fence that needs an instruction is a sequentially-consistent
7762 // cross-thread fence.
7763 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
7764 FenceSSID == SyncScope::System)
7765 return Op;
7766
7767 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7768 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
7769 }
7770
7771 // singlethread fences only synchronize with signal handlers on the same
7772 // thread and thus only need to preserve instruction order, not actually
7773 // enforce memory ordering.
7774 if (FenceSSID == SyncScope::SingleThread)
7775 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7776 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
7777
7778 return Op;
7779}
7780
7781SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
7782 SelectionDAG &DAG) const {
7783 SDLoc DL(Op);
7784 MVT VT = Op.getSimpleValueType();
7785 MVT XLenVT = Subtarget.getXLenVT();
7786 unsigned Check = Op.getConstantOperandVal(i: 1);
7787 unsigned TDCMask = 0;
7788 if (Check & fcSNan)
7789 TDCMask |= RISCV::FPMASK_Signaling_NaN;
7790 if (Check & fcQNan)
7791 TDCMask |= RISCV::FPMASK_Quiet_NaN;
7792 if (Check & fcPosInf)
7793 TDCMask |= RISCV::FPMASK_Positive_Infinity;
7794 if (Check & fcNegInf)
7795 TDCMask |= RISCV::FPMASK_Negative_Infinity;
7796 if (Check & fcPosNormal)
7797 TDCMask |= RISCV::FPMASK_Positive_Normal;
7798 if (Check & fcNegNormal)
7799 TDCMask |= RISCV::FPMASK_Negative_Normal;
7800 if (Check & fcPosSubnormal)
7801 TDCMask |= RISCV::FPMASK_Positive_Subnormal;
7802 if (Check & fcNegSubnormal)
7803 TDCMask |= RISCV::FPMASK_Negative_Subnormal;
7804 if (Check & fcPosZero)
7805 TDCMask |= RISCV::FPMASK_Positive_Zero;
7806 if (Check & fcNegZero)
7807 TDCMask |= RISCV::FPMASK_Negative_Zero;
7808
7809 bool IsOneBitMask = isPowerOf2_32(Value: TDCMask);
7810
7811 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: XLenVT);
7812
7813 if (VT.isVector()) {
7814 SDValue Op0 = Op.getOperand(i: 0);
7815 MVT VT0 = Op.getOperand(i: 0).getSimpleValueType();
7816
7817 if (VT.isScalableVector()) {
7818 MVT DstVT = VT0.changeVectorElementTypeToInteger();
7819 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: VT0, DL, DAG, Subtarget);
7820 SDValue FPCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS_VL, DL, VT: DstVT, N1: Op0, N2: Mask,
7821 N3: VL, Flags: Op->getFlags());
7822 if (IsOneBitMask)
7823 return DAG.getSetCC(DL, VT, LHS: FPCLASS,
7824 RHS: DAG.getConstant(Val: TDCMask, DL, VT: DstVT),
7825 Cond: ISD::CondCode::SETEQ);
7826 SDValue AND = DAG.getNode(Opcode: ISD::AND, DL, VT: DstVT, N1: FPCLASS,
7827 N2: DAG.getConstant(Val: TDCMask, DL, VT: DstVT));
7828 return DAG.getSetCC(DL, VT, LHS: AND, RHS: DAG.getConstant(Val: 0, DL, VT: DstVT),
7829 Cond: ISD::SETNE);
7830 }
7831
7832 MVT ContainerVT0 = getContainerForFixedLengthVector(VT: VT0);
7833 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7834 MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
7835 auto [Mask, VL] = getDefaultVLOps(VecVT: VT0, ContainerVT: ContainerVT0, DL, DAG, Subtarget);
7836 Op0 = convertToScalableVector(VT: ContainerVT0, V: Op0, DAG, Subtarget);
7837
7838 SDValue FPCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS_VL, DL, VT: ContainerDstVT, N1: Op0,
7839 N2: Mask, N3: VL, Flags: Op->getFlags());
7840
7841 TDCMaskV = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerDstVT,
7842 N1: DAG.getUNDEF(VT: ContainerDstVT), N2: TDCMaskV, N3: VL);
7843 if (IsOneBitMask) {
7844 SDValue VMSEQ =
7845 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
7846 Ops: {FPCLASS, TDCMaskV, DAG.getCondCode(Cond: ISD::SETEQ),
7847 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7848 return convertFromScalableVector(VT, V: VMSEQ, DAG, Subtarget);
7849 }
7850 SDValue AND = DAG.getNode(Opcode: RISCVISD::AND_VL, DL, VT: ContainerDstVT, N1: FPCLASS,
7851 N2: TDCMaskV, N3: DAG.getUNDEF(VT: ContainerDstVT), N4: Mask, N5: VL);
7852
7853 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
7854 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerDstVT,
7855 N1: DAG.getUNDEF(VT: ContainerDstVT), N2: SplatZero, N3: VL);
7856
7857 SDValue VMSNE = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
7858 Ops: {AND, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
7859 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7860 return convertFromScalableVector(VT, V: VMSNE, DAG, Subtarget);
7861 }
7862
7863 SDValue FCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS, DL, VT: XLenVT, Operand: Op.getOperand(i: 0));
7864 SDValue AND = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: FCLASS, N2: TDCMaskV);
7865 SDValue Res = DAG.getSetCC(DL, VT: XLenVT, LHS: AND, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT),
7866 Cond: ISD::CondCode::SETNE);
7867 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
7868}
7869
7870// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
7871// operations propagate nans.
7872static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
7873 const RISCVSubtarget &Subtarget) {
7874 SDLoc DL(Op);
7875 MVT VT = Op.getSimpleValueType();
7876
7877 SDValue X = Op.getOperand(i: 0);
7878 SDValue Y = Op.getOperand(i: 1);
7879
7880 if (!VT.isVector()) {
7881 MVT XLenVT = Subtarget.getXLenVT();
7882
7883 // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
7884 // ensures that when one input is a nan, the other will also be a nan
7885 // allowing the nan to propagate. If both inputs are nan, this will swap the
7886 // inputs which is harmless.
7887
7888 SDValue NewY = Y;
7889 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Op: X)) {
7890 SDValue XIsNonNan = DAG.getSetCC(DL, VT: XLenVT, LHS: X, RHS: X, Cond: ISD::SETOEQ);
7891 NewY = DAG.getSelect(DL, VT, Cond: XIsNonNan, LHS: Y, RHS: X);
7892 }
7893
7894 SDValue NewX = X;
7895 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Op: Y)) {
7896 SDValue YIsNonNan = DAG.getSetCC(DL, VT: XLenVT, LHS: Y, RHS: Y, Cond: ISD::SETOEQ);
7897 NewX = DAG.getSelect(DL, VT, Cond: YIsNonNan, LHS: X, RHS: Y);
7898 }
7899
7900 unsigned Opc =
7901 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
7902 return DAG.getNode(Opcode: Opc, DL, VT, N1: NewX, N2: NewY);
7903 }
7904
7905 // Check no NaNs before converting to fixed vector scalable.
7906 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Op: X);
7907 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Op: Y);
7908
7909 MVT ContainerVT = VT;
7910 if (VT.isFixedLengthVector()) {
7911 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
7912 X = convertToScalableVector(VT: ContainerVT, V: X, DAG, Subtarget);
7913 Y = convertToScalableVector(VT: ContainerVT, V: Y, DAG, Subtarget);
7914 }
7915
7916 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
7917
7918 SDValue NewY = Y;
7919 if (!XIsNeverNan) {
7920 SDValue XIsNonNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
7921 Ops: {X, X, DAG.getCondCode(Cond: ISD::SETOEQ),
7922 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7923 NewY = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: XIsNonNan, N2: Y, N3: X,
7924 N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
7925 }
7926
7927 SDValue NewX = X;
7928 if (!YIsNeverNan) {
7929 SDValue YIsNonNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
7930 Ops: {Y, Y, DAG.getCondCode(Cond: ISD::SETOEQ),
7931 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7932 NewX = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: YIsNonNan, N2: X, N3: Y,
7933 N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
7934 }
7935
7936 unsigned Opc =
7937 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
7938 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: NewX, N2: NewY,
7939 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
7940 if (VT.isFixedLengthVector())
7941 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
7942 return Res;
7943}
7944
7945static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
7946 const RISCVSubtarget &Subtarget) {
7947 bool IsFABS = Op.getOpcode() == ISD::FABS;
7948 assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
7949 "Wrong opcode for lowering FABS or FNEG.");
7950
7951 MVT XLenVT = Subtarget.getXLenVT();
7952 MVT VT = Op.getSimpleValueType();
7953 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7954
7955 SDLoc DL(Op);
7956 SDValue Fmv =
7957 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Op.getOperand(i: 0));
7958
7959 APInt Mask = IsFABS ? APInt::getSignedMaxValue(numBits: 16) : APInt::getSignMask(BitWidth: 16);
7960 Mask = Mask.sext(width: Subtarget.getXLen());
7961
7962 unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
7963 SDValue Logic =
7964 DAG.getNode(Opcode: LogicOpc, DL, VT: XLenVT, N1: Fmv, N2: DAG.getConstant(Val: Mask, DL, VT: XLenVT));
7965 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: Logic);
7966}
7967
7968static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
7969 const RISCVSubtarget &Subtarget) {
7970 assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
7971
7972 MVT XLenVT = Subtarget.getXLenVT();
7973 MVT VT = Op.getSimpleValueType();
7974 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7975
7976 SDValue Mag = Op.getOperand(i: 0);
7977 SDValue Sign = Op.getOperand(i: 1);
7978
7979 SDLoc DL(Op);
7980
7981 // Get sign bit into an integer value.
7982 unsigned SignSize = Sign.getValueSizeInBits();
7983 SDValue SignAsInt = [&]() {
7984 if (SignSize == Subtarget.getXLen())
7985 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: XLenVT, Operand: Sign);
7986 switch (SignSize) {
7987 case 16:
7988 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Sign);
7989 case 32:
7990 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: XLenVT, Operand: Sign);
7991 case 64: {
7992 assert(XLenVT == MVT::i32 && "Unexpected type");
7993 // Copy the upper word to integer.
7994 SignSize = 32;
7995 return DAG.getNode(Opcode: RISCVISD::SplitF64, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: Sign)
7996 .getValue(R: 1);
7997 }
7998 default:
7999 llvm_unreachable("Unexpected sign size");
8000 }
8001 }();
8002
8003 // Get the signbit at the right position for MagAsInt.
8004 if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
8005 SignAsInt = DAG.getNode(Opcode: ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, VT: XLenVT,
8006 N1: SignAsInt,
8007 N2: DAG.getConstant(Val: std::abs(x: ShiftAmount), DL, VT: XLenVT));
8008
8009 // Mask the sign bit and any bits above it. The extra bits will be dropped
8010 // when we convert back to FP.
8011 SDValue SignMask = DAG.getConstant(
8012 Val: APInt::getSignMask(BitWidth: 16).sext(width: Subtarget.getXLen()), DL, VT: XLenVT);
8013 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: SignAsInt, N2: SignMask);
8014
8015 // Transform Mag value to integer, and clear the sign bit.
8016 SDValue MagAsInt = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Mag);
8017 SDValue ClearSignMask = DAG.getConstant(
8018 Val: APInt::getSignedMaxValue(numBits: 16).sext(width: Subtarget.getXLen()), DL, VT: XLenVT);
8019 SDValue ClearedSign =
8020 DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: MagAsInt, N2: ClearSignMask);
8021
8022 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: XLenVT, N1: ClearedSign, N2: SignBit,
8023 Flags: SDNodeFlags::Disjoint);
8024
8025 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: CopiedSign);
8026}
8027
8028/// Get a RISC-V target specified VL op for a given SDNode.
8029static unsigned getRISCVVLOp(SDValue Op) {
8030#define OP_CASE(NODE) \
8031 case ISD::NODE: \
8032 return RISCVISD::NODE##_VL;
8033#define VP_CASE(NODE) \
8034 case ISD::VP_##NODE: \
8035 return RISCVISD::NODE##_VL;
8036 // clang-format off
8037 switch (Op.getOpcode()) {
8038 default:
8039 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
8040 OP_CASE(ADD)
8041 OP_CASE(SUB)
8042 OP_CASE(MUL)
8043 OP_CASE(MULHS)
8044 OP_CASE(MULHU)
8045 OP_CASE(SDIV)
8046 OP_CASE(SREM)
8047 OP_CASE(UDIV)
8048 OP_CASE(UREM)
8049 OP_CASE(SHL)
8050 OP_CASE(SRA)
8051 OP_CASE(SRL)
8052 OP_CASE(ROTL)
8053 OP_CASE(ROTR)
8054 OP_CASE(BSWAP)
8055 OP_CASE(CTTZ)
8056 OP_CASE(CTLZ)
8057 OP_CASE(CTPOP)
8058 OP_CASE(BITREVERSE)
8059 OP_CASE(CLMUL)
8060 OP_CASE(CLMULH)
8061 OP_CASE(SADDSAT)
8062 OP_CASE(UADDSAT)
8063 OP_CASE(SSUBSAT)
8064 OP_CASE(USUBSAT)
8065 OP_CASE(AVGFLOORS)
8066 OP_CASE(AVGFLOORU)
8067 OP_CASE(AVGCEILS)
8068 OP_CASE(AVGCEILU)
8069 OP_CASE(FADD)
8070 OP_CASE(FSUB)
8071 OP_CASE(FMUL)
8072 OP_CASE(FDIV)
8073 OP_CASE(FNEG)
8074 OP_CASE(FABS)
8075 OP_CASE(FCOPYSIGN)
8076 OP_CASE(FSQRT)
8077 OP_CASE(SMIN)
8078 OP_CASE(SMAX)
8079 OP_CASE(UMIN)
8080 OP_CASE(UMAX)
8081 OP_CASE(ABDS)
8082 OP_CASE(ABDU)
8083 OP_CASE(STRICT_FADD)
8084 OP_CASE(STRICT_FSUB)
8085 OP_CASE(STRICT_FMUL)
8086 OP_CASE(STRICT_FDIV)
8087 OP_CASE(STRICT_FSQRT)
8088 VP_CASE(SDIV) // VP_SDIV
8089 VP_CASE(SREM) // VP_SREM
8090 VP_CASE(UDIV) // VP_UDIV
8091 VP_CASE(UREM) // VP_UREM
8092 case ISD::CTLZ_ZERO_POISON:
8093 return RISCVISD::CTLZ_VL;
8094 case ISD::CTTZ_ZERO_POISON:
8095 return RISCVISD::CTTZ_VL;
8096 case ISD::FMA:
8097 return RISCVISD::VFMADD_VL;
8098 case ISD::STRICT_FMA:
8099 return RISCVISD::STRICT_VFMADD_VL;
8100 case ISD::AND:
8101 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8102 return RISCVISD::VMAND_VL;
8103 return RISCVISD::AND_VL;
8104 case ISD::OR:
8105 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8106 return RISCVISD::VMOR_VL;
8107 return RISCVISD::OR_VL;
8108 case ISD::XOR:
8109 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
8110 return RISCVISD::VMXOR_VL;
8111 return RISCVISD::XOR_VL;
8112 case ISD::ANY_EXTEND:
8113 case ISD::ZERO_EXTEND:
8114 return RISCVISD::VZEXT_VL;
8115 case ISD::SIGN_EXTEND:
8116 return RISCVISD::VSEXT_VL;
8117 case ISD::SETCC:
8118 return RISCVISD::SETCC_VL;
8119 case ISD::VSELECT:
8120 return RISCVISD::VMERGE_VL;
8121 case ISD::VP_MERGE:
8122 return RISCVISD::VMERGE_VL;
8123 case ISD::FMINNUM:
8124 case ISD::FMINIMUMNUM:
8125 return RISCVISD::VFMIN_VL;
8126 case ISD::FMAXNUM:
8127 case ISD::FMAXIMUMNUM:
8128 return RISCVISD::VFMAX_VL;
8129 case ISD::LRINT:
8130 case ISD::LLRINT:
8131 return RISCVISD::VFCVT_RM_X_F_VL;
8132 case ISD::MASKED_UDIV:
8133 return RISCVISD::UDIV_VL;
8134 case ISD::MASKED_UREM:
8135 return RISCVISD::UREM_VL;
8136 case ISD::MASKED_SDIV:
8137 return RISCVISD::SDIV_VL;
8138 case ISD::MASKED_SREM:
8139 return RISCVISD::SREM_VL;
8140 }
8141 // clang-format on
8142#undef OP_CASE
8143#undef VP_CASE
8144}
8145
8146static bool isPromotedOpNeedingSplit(SDValue Op,
8147 const RISCVSubtarget &Subtarget,
8148 const TargetLowering &TLI) {
8149 MVT OpVT = Op.getSimpleValueType();
8150 if (!OpVT.isVector())
8151 return false;
8152 MVT EltVT = OpVT.getVectorElementType();
8153 if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
8154 !Subtarget.hasVInstructionsF16()) &&
8155 !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
8156 (!Subtarget.hasVInstructionsBF16() ||
8157 !llvm::is_contained(Range: ZvfbfaOps, Element: Op.getOpcode()))))
8158 return false;
8159 // Need to split when the same width f32 vector type isn't legal.
8160 return !TLI.isTypeLegal(
8161 VT: MVT::getVectorVT(VT: MVT::f32, EC: OpVT.getVectorElementCount()));
8162}
8163
8164static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
8165 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op.getValueType());
8166 SDLoc DL(Op);
8167
8168 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8169 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8170
8171 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8172 if (!Op.getOperand(i: j).getValueType().isVector()) {
8173 LoOperands[j] = Op.getOperand(i: j);
8174 HiOperands[j] = Op.getOperand(i: j);
8175 continue;
8176 }
8177 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
8178 DAG.SplitVector(N: Op.getOperand(i: j), DL);
8179 }
8180
8181 SDValue LoRes =
8182 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: LoVT, Ops: LoOperands, Flags: Op->getFlags());
8183 SDValue HiRes =
8184 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: HiVT, Ops: HiOperands, Flags: Op->getFlags());
8185
8186 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op.getValueType(), N1: LoRes, N2: HiRes);
8187}
8188
8189static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
8190 bool IsVP) {
8191 SDLoc DL(Op);
8192
8193 if (IsVP) {
8194 auto [Lo, Hi] = DAG.SplitVector(N: Op.getOperand(i: 1), DL);
8195 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op.getOperand(i: 2), DL);
8196 auto [EVLLo, EVLHi] =
8197 DAG.SplitEVL(N: Op.getOperand(i: 3), VecVT: Op.getOperand(i: 1).getValueType(), DL);
8198
8199 SDValue ResLo =
8200 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
8201 Ops: {Op.getOperand(i: 0), Lo, MaskLo, EVLLo}, Flags: Op->getFlags());
8202 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
8203 Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags: Op->getFlags());
8204 }
8205
8206 unsigned Opcode = Op.getOpcode();
8207 unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
8208
8209 auto [Lo, Hi] = DAG.SplitVector(N: Op.getOperand(i: OpNo), DL);
8210 if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
8211 SDValue ResLo = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
8212 N1: Op.getOperand(i: 0), N2: Lo, Flags: Op->getFlags());
8213 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), N1: ResLo, N2: Hi,
8214 Flags: Op->getFlags());
8215 }
8216
8217 SDValue ResLo =
8218 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Lo, Flags: Op->getFlags());
8219 SDValue ResHi =
8220 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Hi, Flags: Op->getFlags());
8221 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
8222 return DAG.getNode(Opcode: BaseOpc, DL, VT: Op.getValueType(), N1: ResLo, N2: ResHi,
8223 Flags: Op->getFlags());
8224}
8225
8226static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
8227
8228 assert(Op->isStrictFPOpcode());
8229
8230 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op->getValueType(ResNo: 0));
8231
8232 SDVTList LoVTs = DAG.getVTList(VT1: LoVT, VT2: Op->getValueType(ResNo: 1));
8233 SDVTList HiVTs = DAG.getVTList(VT1: HiVT, VT2: Op->getValueType(ResNo: 1));
8234
8235 SDLoc DL(Op);
8236
8237 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8238 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8239
8240 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8241 if (!Op.getOperand(i: j).getValueType().isVector()) {
8242 LoOperands[j] = Op.getOperand(i: j);
8243 HiOperands[j] = Op.getOperand(i: j);
8244 continue;
8245 }
8246 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
8247 DAG.SplitVector(N: Op.getOperand(i: j), DL);
8248 }
8249
8250 SDValue LoRes =
8251 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: LoVTs, Ops: LoOperands, Flags: Op->getFlags());
8252 HiOperands[0] = LoRes.getValue(R: 1);
8253 SDValue HiRes =
8254 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: HiVTs, Ops: HiOperands, Flags: Op->getFlags());
8255
8256 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op->getValueType(ResNo: 0),
8257 N1: LoRes.getValue(R: 0), N2: HiRes.getValue(R: 0));
8258 return DAG.getMergeValues(Ops: {V, HiRes.getValue(R: 1)}, dl: DL);
8259}
8260
8261SDValue
8262RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
8263 SelectionDAG &DAG) const {
8264 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8265 "Unexpected bfloat16 load lowering");
8266
8267 SDLoc DL(Op);
8268 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
8269 EVT MemVT = LD->getMemoryVT();
8270 SDValue Load = DAG.getExtLoad(
8271 ExtType: ISD::ZEXTLOAD, dl: DL, VT: Subtarget.getXLenVT(), Chain: LD->getChain(),
8272 Ptr: LD->getBasePtr(),
8273 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
8274 MMO: LD->getMemOperand());
8275 // Using mask to make bf16 nan-boxing valid when we don't have flh
8276 // instruction. -65536 would be treat as a small number and thus it can be
8277 // directly used lui to get the constant.
8278 SDValue mask = DAG.getSignedConstant(Val: -65536, DL, VT: Subtarget.getXLenVT());
8279 SDValue OrSixteenOne =
8280 DAG.getNode(Opcode: ISD::OR, DL, VT: Load.getValueType(), Ops: {Load, mask});
8281 SDValue ConvertedResult =
8282 DAG.getNode(Opcode: RISCVISD::NDS_FMV_BF16_X, DL, VT: MVT::bf16, Operand: OrSixteenOne);
8283 return DAG.getMergeValues(Ops: {ConvertedResult, Load.getValue(R: 1)}, dl: DL);
8284}
8285
8286SDValue
8287RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
8288 SelectionDAG &DAG) const {
8289 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8290 "Unexpected bfloat16 store lowering");
8291
8292 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
8293 SDLoc DL(Op);
8294 SDValue FMV = DAG.getNode(Opcode: RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
8295 VT: Subtarget.getXLenVT(), Operand: ST->getValue());
8296 return DAG.getTruncStore(
8297 Chain: ST->getChain(), dl: DL, Val: FMV, Ptr: ST->getBasePtr(),
8298 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits()),
8299 MMO: ST->getMemOperand());
8300}
8301
8302static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
8303 const RISCVSubtarget &Subtarget);
8304
8305static SDValue lowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG,
8306 const RISCVSubtarget &Subtarget);
8307
8308SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
8309 SelectionDAG &DAG) const {
8310 switch (Op.getOpcode()) {
8311 default:
8312 reportFatalInternalError(
8313 reason: "Unimplemented RISCVTargetLowering::LowerOperation Case");
8314 case ISD::PREFETCH:
8315 return LowerPREFETCH(Op, Subtarget, DAG);
8316 case ISD::ATOMIC_FENCE:
8317 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
8318 case ISD::GlobalAddress:
8319 return lowerGlobalAddress(Op, DAG);
8320 case ISD::BlockAddress:
8321 return lowerBlockAddress(Op, DAG);
8322 case ISD::ConstantPool:
8323 return lowerConstantPool(Op, DAG);
8324 case ISD::JumpTable:
8325 return lowerJumpTable(Op, DAG);
8326 case ISD::GlobalTLSAddress:
8327 return lowerGlobalTLSAddress(Op, DAG);
8328 case ISD::Constant:
8329 return lowerConstant(Op, DAG, Subtarget);
8330 case ISD::ConstantFP:
8331 return lowerConstantFP(Op, DAG);
8332 case ISD::SELECT:
8333 return lowerSELECT(Op, DAG);
8334 case ISD::BRCOND:
8335 return lowerBRCOND(Op, DAG);
8336 case ISD::VASTART:
8337 return lowerVASTART(Op, DAG);
8338 case ISD::FRAMEADDR:
8339 return lowerFRAMEADDR(Op, DAG);
8340 case ISD::RETURNADDR:
8341 return lowerRETURNADDR(Op, DAG);
8342 case ISD::SHL_PARTS:
8343 return lowerShiftLeftParts(Op, DAG);
8344 case ISD::SRA_PARTS:
8345 return lowerShiftRightParts(Op, DAG, IsSRA: true);
8346 case ISD::SRL_PARTS:
8347 return lowerShiftRightParts(Op, DAG, IsSRA: false);
8348 case ISD::ROTL:
8349 case ISD::ROTR:
8350 if (Op.getValueType().isFixedLengthVector()) {
8351 assert(Subtarget.hasStdExtZvkb());
8352 return lowerToScalableOp(Op, DAG);
8353 }
8354 assert(Subtarget.hasVendorXTHeadBb() &&
8355 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
8356 "Unexpected custom legalization");
8357 // XTHeadBb only supports rotate by constant.
8358 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
8359 return SDValue();
8360 return Op;
8361 case ISD::BITCAST: {
8362 SDLoc DL(Op);
8363 EVT VT = Op.getValueType();
8364 SDValue Op0 = Op.getOperand(i: 0);
8365 EVT Op0VT = Op0.getValueType();
8366 MVT XLenVT = Subtarget.getXLenVT();
8367 if (Op0VT == MVT::i16 &&
8368 ((VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
8369 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
8370 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Op0);
8371 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: NewOp0);
8372 }
8373 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
8374 Subtarget.hasStdExtFOrZfinx()) {
8375 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
8376 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: NewOp0);
8377 }
8378 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() &&
8379 Subtarget.hasStdExtDOrZdinx()) {
8380 SDValue Lo, Hi;
8381 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
8382 return DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
8383 }
8384
8385 if (Subtarget.hasStdExtP() && VT.isSimple() && Op0VT.isSimple()) {
8386 if (VT.getSimpleVT() == Subtarget.getXLenVT() &&
8387 Subtarget.isPExtPackedType(VT: Op0VT.getSimpleVT()))
8388 return Op;
8389 if (Op0VT.getSimpleVT() == Subtarget.getXLenVT() &&
8390 Subtarget.isPExtPackedType(VT: VT.getSimpleVT()))
8391 return Op;
8392 }
8393
8394 // Consider other scalar<->scalar casts as legal if the types are legal.
8395 // Otherwise expand them.
8396 if (!VT.isVector() && !Op0VT.isVector()) {
8397 if (isTypeLegal(VT) && isTypeLegal(VT: Op0VT))
8398 return Op;
8399 return SDValue();
8400 }
8401
8402 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
8403 "Unexpected types");
8404
8405 if (VT.isFixedLengthVector()) {
8406 // We can handle fixed length vector bitcasts with a simple replacement
8407 // in isel.
8408 if (Op0VT.isFixedLengthVector())
8409 return Op;
8410 // When bitcasting from scalar to fixed-length vector, insert the scalar
8411 // into a one-element vector of the result type, and perform a vector
8412 // bitcast.
8413 if (!Op0VT.isVector()) {
8414 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: Op0VT, NumElements: 1);
8415 if (!isTypeLegal(VT: BVT))
8416 return SDValue();
8417 return DAG.getBitcast(
8418 VT, V: DAG.getInsertVectorElt(DL, Vec: DAG.getUNDEF(VT: BVT), Elt: Op0, Idx: 0));
8419 }
8420 return SDValue();
8421 }
8422 // Custom-legalize bitcasts from fixed-length vector types to scalar types
8423 // thus: bitcast the vector to a one-element vector type whose element type
8424 // is the same as the result type, and extract the first element.
8425 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
8426 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 1);
8427 if (!isTypeLegal(VT: BVT))
8428 return SDValue();
8429 SDValue BVec = DAG.getBitcast(VT: BVT, V: Op0);
8430 return DAG.getExtractVectorElt(DL, VT, Vec: BVec, Idx: 0);
8431 }
8432 return SDValue();
8433 }
8434 case ISD::INTRINSIC_WO_CHAIN:
8435 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
8436 case ISD::INTRINSIC_W_CHAIN:
8437 return LowerINTRINSIC_W_CHAIN(Op, DAG);
8438 case ISD::INTRINSIC_VOID:
8439 return LowerINTRINSIC_VOID(Op, DAG);
8440 case ISD::IS_FPCLASS:
8441 return LowerIS_FPCLASS(Op, DAG);
8442 case ISD::BITREVERSE: {
8443 MVT VT = Op.getSimpleValueType();
8444 if (VT.isFixedLengthVector()) {
8445 assert(Subtarget.hasStdExtZvbb());
8446 return lowerToScalableOp(Op, DAG);
8447 }
8448 SDLoc DL(Op);
8449 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
8450 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
8451 // Expand bitreverse to a bswap(rev8) followed by brev8.
8452 SDValue BSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT, Operand: Op.getOperand(i: 0));
8453 return DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT, Operand: BSwap);
8454 }
8455 case ISD::TRUNCATE:
8456 case ISD::TRUNCATE_SSAT_S:
8457 case ISD::TRUNCATE_USAT_U:
8458 // Only custom-lower vector truncates
8459 if (!Op.getSimpleValueType().isVector())
8460 return Op;
8461 return lowerVectorTrunc(Op, DAG);
8462 case ISD::ANY_EXTEND:
8463 case ISD::ZERO_EXTEND:
8464 if (Op.getOperand(i: 0).getValueType().isVector() &&
8465 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8466 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ ExtTrueVal: 1);
8467 if (Op.getValueType().isScalableVector())
8468 return Op;
8469 return lowerToScalableOp(Op, DAG);
8470 case ISD::SIGN_EXTEND:
8471 if (Op.getOperand(i: 0).getValueType().isVector() &&
8472 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8473 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ ExtTrueVal: -1);
8474 if (Op.getValueType().isScalableVector())
8475 return Op;
8476 return lowerToScalableOp(Op, DAG);
8477 case ISD::SPLAT_VECTOR_PARTS:
8478 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
8479 case ISD::INSERT_VECTOR_ELT:
8480 return lowerINSERT_VECTOR_ELT(Op, DAG);
8481 case ISD::EXTRACT_VECTOR_ELT:
8482 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
8483 case ISD::SCALAR_TO_VECTOR: {
8484 MVT VT = Op.getSimpleValueType();
8485 SDLoc DL(Op);
8486 SDValue Scalar = Op.getOperand(i: 0);
8487 if (VT.getVectorElementType() == MVT::i1) {
8488 MVT WideVT = VT.changeVectorElementType(EltVT: MVT::i8);
8489 SDValue V = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: WideVT, Operand: Scalar);
8490 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: V);
8491 }
8492 MVT ContainerVT = VT;
8493 if (VT.isFixedLengthVector())
8494 ContainerVT = getContainerForFixedLengthVector(VT);
8495 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
8496
8497 SDValue V;
8498 if (VT.isFloatingPoint()) {
8499 V = DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT: ContainerVT,
8500 N1: DAG.getUNDEF(VT: ContainerVT), N2: Scalar, N3: VL);
8501 } else {
8502 Scalar = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: Scalar);
8503 V = DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT: ContainerVT,
8504 N1: DAG.getUNDEF(VT: ContainerVT), N2: Scalar, N3: VL);
8505 }
8506 if (VT.isFixedLengthVector())
8507 V = convertFromScalableVector(VT, V, DAG, Subtarget);
8508 return V;
8509 }
8510 case ISD::VSCALE: {
8511 MVT XLenVT = Subtarget.getXLenVT();
8512 MVT VT = Op.getSimpleValueType();
8513 SDLoc DL(Op);
8514 SDValue Res = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
8515 // We define our scalable vector types for lmul=1 to use a 64 bit known
8516 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
8517 // vscale as VLENB / 8.
8518 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
8519 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
8520 reportFatalInternalError(reason: "Support for VLEN==32 is incomplete.");
8521 // We assume VLENB is a multiple of 8. We manually choose the best shift
8522 // here because SimplifyDemandedBits isn't always able to simplify it.
8523 uint64_t Val = Op.getConstantOperandVal(i: 0);
8524 if (isPowerOf2_64(Value: Val)) {
8525 uint64_t Log2 = Log2_64(Value: Val);
8526 if (Log2 < 3) {
8527 SDNodeFlags Flags;
8528 Flags.setExact(true);
8529 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Res,
8530 N2: DAG.getConstant(Val: 3 - Log2, DL, VT: XLenVT), Flags);
8531 } else if (Log2 > 3) {
8532 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: Res,
8533 N2: DAG.getConstant(Val: Log2 - 3, DL, VT: XLenVT));
8534 }
8535 } else if ((Val % 8) == 0) {
8536 // If the multiplier is a multiple of 8, scale it down to avoid needing
8537 // to shift the VLENB value.
8538 Res = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Res,
8539 N2: DAG.getConstant(Val: Val / 8, DL, VT: XLenVT));
8540 } else {
8541 SDNodeFlags Flags;
8542 Flags.setExact(true);
8543 SDValue VScale = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Res,
8544 N2: DAG.getConstant(Val: 3, DL, VT: XLenVT), Flags);
8545 Res = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: VScale,
8546 N2: DAG.getConstant(Val, DL, VT: XLenVT));
8547 }
8548 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
8549 }
8550 case ISD::FPOWI: {
8551 // Custom promote f16 powi with illegal i32 integer type on RV64. Once
8552 // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
8553 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
8554 Op.getOperand(i: 1).getValueType() == MVT::i32) {
8555 SDLoc DL(Op);
8556 SDValue Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op.getOperand(i: 0));
8557 SDValue Powi =
8558 DAG.getNode(Opcode: ISD::FPOWI, DL, VT: MVT::f32, N1: Op0, N2: Op.getOperand(i: 1));
8559 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Powi,
8560 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
8561 }
8562 return SDValue();
8563 }
8564 case ISD::FMAXIMUM:
8565 case ISD::FMINIMUM:
8566 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
8567 return SplitVectorOp(Op, DAG);
8568 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
8569 case ISD::FP_EXTEND:
8570 case ISD::FP_ROUND:
8571 return lowerVectorFPExtendOrRound(Op, DAG);
8572 case ISD::STRICT_FP_ROUND:
8573 case ISD::STRICT_FP_EXTEND:
8574 return lowerStrictFPExtendOrRoundLike(Op, DAG);
8575 case ISD::SINT_TO_FP:
8576 case ISD::UINT_TO_FP:
8577 // An i1-to-fp conversion is equivalent to selecting between 1.0/-1.0 and
8578 // 0.0 for unsigned/signed conversion, respectively. This avoids extending
8579 // the mask before converting it.
8580 if (Op.getValueType().isVector() &&
8581 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1) {
8582 SDLoc DL(Op);
8583 EVT VT = Op.getValueType();
8584 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL, VT);
8585 double TrueVal = Op.getOpcode() == ISD::UINT_TO_FP ? 1.0 : -1.0;
8586 SDValue True = DAG.getConstantFP(Val: TrueVal, DL, VT);
8587 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: Op.getOperand(i: 0), N2: True, N3: Zero);
8588 }
8589 // Fall back to zvfbfmin for bf16 case if source type is wider than 8 bits.
8590 if (SDValue Op1 = Op.getOperand(i: 0);
8591 Op.getValueType().isVector() &&
8592 ((Op.getValueType().getScalarType() == MVT::f16 &&
8593 (Subtarget.hasVInstructionsF16Minimal() &&
8594 !Subtarget.hasVInstructionsF16())) ||
8595 (Op.getValueType().getScalarType() == MVT::bf16 &&
8596 (Subtarget.hasVInstructionsBF16Minimal() &&
8597 (!Subtarget.hasVInstructionsBF16() ||
8598 Op1.getValueType().getScalarSizeInBits() > 8))))) {
8599 MVT NVT =
8600 MVT::getVectorVT(VT: MVT::f32, EC: Op.getValueType().getVectorElementCount());
8601 if (!isTypeLegal(VT: NVT))
8602 return SplitVectorOp(Op, DAG);
8603 // int -> f32
8604 SDLoc DL(Op);
8605 SDValue NC = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: NVT, Ops: Op->ops());
8606 // f32 -> [b]f16
8607 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(), N1: NC,
8608 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
8609 }
8610 [[fallthrough]];
8611 case ISD::FP_TO_SINT:
8612 case ISD::FP_TO_UINT:
8613 // Fall back to zvfbfmin for bf16 case if destination type is wider than 8
8614 // bits.
8615 if (SDValue Op1 = Op.getOperand(i: 0);
8616 Op1.getValueType().isVector() &&
8617 ((Op1.getValueType().getScalarType() == MVT::f16 &&
8618 (Subtarget.hasVInstructionsF16Minimal() &&
8619 !Subtarget.hasVInstructionsF16())) ||
8620 (Op1.getValueType().getScalarType() == MVT::bf16 &&
8621 (Subtarget.hasVInstructionsBF16Minimal() &&
8622 (!Subtarget.hasVInstructionsBF16() ||
8623 Op.getValueType().getScalarSizeInBits() > 8))))) {
8624 MVT NVT = MVT::getVectorVT(VT: MVT::f32,
8625 EC: Op1.getValueType().getVectorElementCount());
8626 if (!isTypeLegal(VT: NVT))
8627 return SplitVectorOp(Op, DAG);
8628 // [b]f16 -> f32
8629 SDLoc DL(Op);
8630 SDValue WidenVec = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NVT, Operand: Op1);
8631 // f32 -> int
8632 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: WidenVec);
8633 }
8634 [[fallthrough]];
8635 case ISD::STRICT_FP_TO_SINT:
8636 case ISD::STRICT_FP_TO_UINT:
8637 case ISD::STRICT_SINT_TO_FP:
8638 case ISD::STRICT_UINT_TO_FP: {
8639 // RVV can only do fp<->int conversions to types half/double the size as
8640 // the source. We custom-lower any conversions that do two hops into
8641 // sequences.
8642 MVT VT = Op.getSimpleValueType();
8643 if (VT.isScalarInteger())
8644 return lowerFP_TO_INT(Op, DAG, Subtarget);
8645 bool IsStrict = Op->isStrictFPOpcode();
8646 SDValue Src = Op.getOperand(i: 0 + IsStrict);
8647 MVT SrcVT = Src.getSimpleValueType();
8648 if (SrcVT.isScalarInteger())
8649 return lowerINT_TO_FP(Op, DAG, Subtarget);
8650 if (!VT.isVector())
8651 return Op;
8652 SDLoc DL(Op);
8653 MVT EltVT = VT.getVectorElementType();
8654 MVT SrcEltVT = SrcVT.getVectorElementType();
8655 unsigned EltSize = EltVT.getSizeInBits();
8656 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
8657 assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
8658 "Unexpected vector element types");
8659
8660 bool IsInt2FP = SrcEltVT.isInteger();
8661 // Widening conversions
8662 if (EltSize > (2 * SrcEltSize)) {
8663 if (IsInt2FP) {
8664 // Do a regular integer sign/zero extension then convert to float.
8665 MVT IVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSize / 2),
8666 EC: VT.getVectorElementCount());
8667 unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP ||
8668 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
8669 ? ISD::ZERO_EXTEND
8670 : ISD::SIGN_EXTEND;
8671 SDValue Ext = DAG.getNode(Opcode: ExtOpcode, DL, VT: IVecVT, Operand: Src);
8672 if (IsStrict)
8673 return DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: Op->getVTList(),
8674 N1: Op.getOperand(i: 0), N2: Ext);
8675 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, Operand: Ext);
8676 }
8677 // FP2Int
8678 assert((SrcEltVT == MVT::f16 || SrcEltVT == MVT::bf16) &&
8679 "Unexpected FP_TO_[US]INT lowering");
8680 // Do one doubling fp_extend then complete the operation by converting
8681 // to int.
8682 MVT InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
8683 if (IsStrict) {
8684 auto [FExt, Chain] =
8685 DAG.getStrictFPExtendOrRound(Op: Src, Chain: Op.getOperand(i: 0), DL, VT: InterimFVT);
8686 return DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: Op->getVTList(), N1: Chain, N2: FExt);
8687 }
8688 SDValue FExt = DAG.getFPExtendOrRound(Op: Src, DL, VT: InterimFVT);
8689 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, Operand: FExt);
8690 }
8691
8692 // Narrowing conversions
8693 if (SrcEltSize > (2 * EltSize)) {
8694 if (IsInt2FP) {
8695 // One narrowing int_to_fp, then an fp_round.
8696 assert((EltVT == MVT::f16 || EltVT == MVT::bf16) &&
8697 "Unexpected [US]_TO_FP lowering");
8698 MVT InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
8699 if (IsStrict) {
8700 SDValue Int2FP = DAG.getNode(Opcode: Op.getOpcode(), DL,
8701 VTList: DAG.getVTList(VT1: InterimFVT, VT2: MVT::Other),
8702 N1: Op.getOperand(i: 0), N2: Src);
8703 SDValue Chain = Int2FP.getValue(R: 1);
8704 return DAG.getStrictFPExtendOrRound(Op: Int2FP, Chain, DL, VT).first;
8705 }
8706 SDValue Int2FP = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: InterimFVT, Operand: Src);
8707 return DAG.getFPExtendOrRound(Op: Int2FP, DL, VT);
8708 }
8709 // FP2Int
8710 // One narrowing fp_to_int, then truncate the integer. If the float isn't
8711 // representable by the integer, the result is poison.
8712 MVT IVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltSize / 2),
8713 EC: VT.getVectorElementCount());
8714 if (IsStrict) {
8715 SDValue FP2Int =
8716 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: DAG.getVTList(VT1: IVecVT, VT2: MVT::Other),
8717 N1: Op.getOperand(i: 0), N2: Src);
8718 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FP2Int);
8719 return DAG.getMergeValues(Ops: {Res, FP2Int.getValue(R: 1)}, dl: DL);
8720 }
8721 SDValue FP2Int = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: IVecVT, Operand: Src);
8722 if (EltSize == 1)
8723 // The integer should be 0 or 1/-1, so compare the integer result to 0.
8724 return DAG.getSetCC(DL, VT, LHS: DAG.getConstant(Val: 0, DL, VT: IVecVT), RHS: FP2Int,
8725 Cond: ISD::SETNE);
8726 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FP2Int);
8727 }
8728
8729 // Scalable vectors can exit here. Patterns will handle equally-sized
8730 // conversions halving/doubling ones.
8731 if (!VT.isFixedLengthVector())
8732 return Op;
8733
8734 // For fixed-length vectors we lower to a custom "VL" node.
8735 unsigned RVVOpc = 0;
8736 switch (Op.getOpcode()) {
8737 default:
8738 llvm_unreachable("Impossible opcode");
8739 case ISD::FP_TO_SINT:
8740 RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
8741 break;
8742 case ISD::FP_TO_UINT:
8743 RVVOpc = RISCVISD::VFCVT_RTZ_XU_F_VL;
8744 break;
8745 case ISD::SINT_TO_FP:
8746 RVVOpc = RISCVISD::SINT_TO_FP_VL;
8747 break;
8748 case ISD::UINT_TO_FP:
8749 RVVOpc = RISCVISD::UINT_TO_FP_VL;
8750 break;
8751 case ISD::STRICT_FP_TO_SINT:
8752 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_X_F_VL;
8753 break;
8754 case ISD::STRICT_FP_TO_UINT:
8755 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL;
8756 break;
8757 case ISD::STRICT_SINT_TO_FP:
8758 RVVOpc = RISCVISD::STRICT_SINT_TO_FP_VL;
8759 break;
8760 case ISD::STRICT_UINT_TO_FP:
8761 RVVOpc = RISCVISD::STRICT_UINT_TO_FP_VL;
8762 break;
8763 }
8764
8765 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8766 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
8767 assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() &&
8768 "Expected same element count");
8769
8770 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
8771
8772 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
8773 if (IsStrict) {
8774 Src = DAG.getNode(Opcode: RVVOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
8775 N1: Op.getOperand(i: 0), N2: Src, N3: Mask, N4: VL);
8776 SDValue SubVec = convertFromScalableVector(VT, V: Src, DAG, Subtarget);
8777 return DAG.getMergeValues(Ops: {SubVec, Src.getValue(R: 1)}, dl: DL);
8778 }
8779 Src = DAG.getNode(Opcode: RVVOpc, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
8780 return convertFromScalableVector(VT, V: Src, DAG, Subtarget);
8781 }
8782 case ISD::FP_TO_SINT_SAT:
8783 case ISD::FP_TO_UINT_SAT:
8784 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
8785 case ISD::FP_TO_BF16: {
8786 // Custom lower to ensure the libcall return is passed in an FPR on hard
8787 // float ABIs.
8788 assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization");
8789 SDLoc DL(Op);
8790 MakeLibCallOptions CallOptions;
8791 RTLIB::Libcall LC =
8792 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
8793 SDValue Res =
8794 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
8795 if (Subtarget.is64Bit())
8796 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Res);
8797 return DAG.getBitcast(VT: MVT::i32, V: Res);
8798 }
8799 case ISD::BF16_TO_FP: {
8800 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization");
8801 MVT VT = Op.getSimpleValueType();
8802 SDLoc DL(Op);
8803 Op = DAG.getNode(
8804 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
8805 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
8806 SDValue Res = Subtarget.is64Bit()
8807 ? DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Op)
8808 : DAG.getBitcast(VT: MVT::f32, V: Op);
8809 // fp_extend if the target VT is bigger than f32.
8810 if (VT != MVT::f32)
8811 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
8812 return Res;
8813 }
8814 case ISD::STRICT_FP_TO_FP16:
8815 case ISD::FP_TO_FP16: {
8816 // Custom lower to ensure the libcall return is passed in an FPR on hard
8817 // float ABIs.
8818 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8819 SDLoc DL(Op);
8820 MakeLibCallOptions CallOptions;
8821 bool IsStrict = Op->isStrictFPOpcode();
8822 SDValue Op0 = IsStrict ? Op.getOperand(i: 1) : Op.getOperand(i: 0);
8823 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8824 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
8825 SDValue Res;
8826 std::tie(args&: Res, args&: Chain) =
8827 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
8828 if (Subtarget.is64Bit())
8829 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Res);
8830 SDValue Result = DAG.getBitcast(VT: MVT::i32, V: IsStrict ? Res.getValue(R: 0) : Res);
8831 if (IsStrict)
8832 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
8833 return Result;
8834 }
8835 case ISD::STRICT_FP16_TO_FP:
8836 case ISD::FP16_TO_FP: {
8837 // Custom lower to ensure the libcall argument is passed in an FPR on hard
8838 // float ABIs.
8839 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8840 SDLoc DL(Op);
8841 MakeLibCallOptions CallOptions;
8842 bool IsStrict = Op->isStrictFPOpcode();
8843 SDValue Op0 = IsStrict ? Op.getOperand(i: 1) : Op.getOperand(i: 0);
8844 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8845 SDValue Arg = Subtarget.is64Bit()
8846 ? DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Op0)
8847 : DAG.getBitcast(VT: MVT::f32, V: Op0);
8848 SDValue Res;
8849 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
8850 CallOptions, dl: DL, Chain);
8851 if (IsStrict)
8852 return DAG.getMergeValues(Ops: {Res, Chain}, dl: DL);
8853 return Res;
8854 }
8855 case ISD::FTRUNC:
8856 case ISD::FCEIL:
8857 case ISD::FFLOOR:
8858 case ISD::FNEARBYINT:
8859 case ISD::FRINT:
8860 case ISD::FROUND:
8861 case ISD::FROUNDEVEN:
8862 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
8863 return SplitVectorOp(Op, DAG);
8864 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
8865 case ISD::FCANONICALIZE: {
8866 MVT VT = Op.getSimpleValueType();
8867 assert(VT.isFixedLengthVector() && "Unexpected type");
8868 SDLoc DL(Op);
8869 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8870 SDValue Src =
8871 convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i: 0), DAG, Subtarget);
8872 SDValue Res = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT: ContainerVT, Operand: Src);
8873 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
8874 }
8875 case ISD::LRINT:
8876 case ISD::LLRINT:
8877 case ISD::LROUND:
8878 case ISD::LLROUND: {
8879 if (Op.getValueType().isVector())
8880 return lowerVectorXRINT_XROUND(Op, DAG, Subtarget);
8881 assert(Op.getOperand(0).getValueType() == MVT::f16 &&
8882 "Unexpected custom legalisation");
8883 SDLoc DL(Op);
8884 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op.getOperand(i: 0));
8885 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Ext);
8886 }
8887 case ISD::STRICT_LRINT:
8888 case ISD::STRICT_LLRINT:
8889 case ISD::STRICT_LROUND:
8890 case ISD::STRICT_LLROUND: {
8891 assert(Op.getOperand(1).getValueType() == MVT::f16 &&
8892 "Unexpected custom legalisation");
8893 SDLoc DL(Op);
8894 SDValue Ext = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
8895 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
8896 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
8897 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
8898 }
8899 case ISD::VECREDUCE_ADD:
8900 case ISD::VECREDUCE_UMAX:
8901 case ISD::VECREDUCE_SMAX:
8902 case ISD::VECREDUCE_UMIN:
8903 case ISD::VECREDUCE_SMIN:
8904 return lowerVECREDUCE(Op, DAG);
8905 case ISD::VECREDUCE_AND:
8906 case ISD::VECREDUCE_OR:
8907 case ISD::VECREDUCE_XOR:
8908 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8909 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
8910 return lowerVECREDUCE(Op, DAG);
8911 case ISD::VECREDUCE_SEQ_FADD:
8912 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 1), Subtarget, TLI: *this))
8913 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
8914 return lowerFPVECREDUCE(Op, DAG);
8915 case ISD::VECREDUCE_FADD:
8916 case ISD::VECREDUCE_FMIN:
8917 case ISD::VECREDUCE_FMAX:
8918 case ISD::VECREDUCE_FMAXIMUM:
8919 case ISD::VECREDUCE_FMINIMUM:
8920 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 0), Subtarget, TLI: *this))
8921 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
8922 return lowerFPVECREDUCE(Op, DAG);
8923 case ISD::VP_REDUCE_ADD:
8924 case ISD::VP_REDUCE_UMAX:
8925 case ISD::VP_REDUCE_SMAX:
8926 case ISD::VP_REDUCE_UMIN:
8927 case ISD::VP_REDUCE_SMIN:
8928 case ISD::VP_REDUCE_FADD:
8929 case ISD::VP_REDUCE_SEQ_FADD:
8930 case ISD::VP_REDUCE_FMIN:
8931 case ISD::VP_REDUCE_FMAX:
8932 case ISD::VP_REDUCE_FMINIMUM:
8933 case ISD::VP_REDUCE_FMAXIMUM:
8934 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 1), Subtarget, TLI: *this))
8935 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ true);
8936 return lowerVPREDUCE(Op, DAG);
8937 case ISD::VP_REDUCE_AND:
8938 case ISD::VP_REDUCE_OR:
8939 case ISD::VP_REDUCE_XOR:
8940 if (Op.getOperand(i: 1).getValueType().getVectorElementType() == MVT::i1)
8941 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
8942 return lowerVPREDUCE(Op, DAG);
8943 case ISD::VP_CTTZ_ELTS:
8944 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8945 return lowerVPCttzElements(Op, DAG);
8946 case ISD::UNDEF:
8947 case ISD::POISON: {
8948 MVT ContainerVT = getContainerForFixedLengthVector(VT: Op.getSimpleValueType());
8949 SDValue Passthru = Op.getOpcode() == ISD::POISON
8950 ? DAG.getPOISON(VT: ContainerVT)
8951 : DAG.getUNDEF(VT: ContainerVT);
8952 return convertFromScalableVector(VT: Op.getSimpleValueType(), V: Passthru, DAG,
8953 Subtarget);
8954 }
8955 case ISD::INSERT_SUBVECTOR:
8956 return lowerINSERT_SUBVECTOR(Op, DAG);
8957 case ISD::EXTRACT_SUBVECTOR:
8958 return lowerEXTRACT_SUBVECTOR(Op, DAG);
8959 case ISD::VECTOR_DEINTERLEAVE:
8960 return lowerVECTOR_DEINTERLEAVE(Op, DAG);
8961 case ISD::VECTOR_INTERLEAVE:
8962 return lowerVECTOR_INTERLEAVE(Op, DAG);
8963 case ISD::STEP_VECTOR:
8964 return lowerSTEP_VECTOR(Op, DAG);
8965 case ISD::VECTOR_REVERSE:
8966 return lowerVECTOR_REVERSE(Op, DAG);
8967 case ISD::VECTOR_SPLICE_LEFT:
8968 case ISD::VECTOR_SPLICE_RIGHT:
8969 return lowerVECTOR_SPLICE(Op, DAG);
8970 case ISD::BUILD_VECTOR: {
8971 MVT VT = Op.getSimpleValueType();
8972 MVT EltVT = VT.getVectorElementType();
8973 if (!Subtarget.is64Bit() && EltVT == MVT::i64)
8974 return lowerBuildVectorViaVID(Op, DAG, Subtarget);
8975 return lowerBUILD_VECTOR(Op, DAG, Subtarget);
8976 }
8977 case ISD::SPLAT_VECTOR: {
8978 MVT VT = Op.getSimpleValueType();
8979 MVT EltVT = VT.getVectorElementType();
8980 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
8981 EltVT == MVT::bf16) {
8982 SDLoc DL(Op);
8983 SDValue Elt;
8984 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
8985 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
8986 Elt = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: Subtarget.getXLenVT(),
8987 Operand: Op.getOperand(i: 0));
8988 else
8989 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op.getOperand(i: 0));
8990 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
8991 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
8992 Operand: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: IVT, Operand: Elt));
8993 }
8994
8995 if (EltVT == MVT::i1)
8996 return lowerVectorMaskSplat(Op, DAG);
8997 return SDValue();
8998 }
8999 case ISD::VECTOR_SHUFFLE:
9000 return lowerVECTOR_SHUFFLE(Op, DAG);
9001 case ISD::CONCAT_VECTORS: {
9002 // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
9003 // better than going through the stack, as the default expansion does.
9004 SDLoc DL(Op);
9005 MVT VT = Op.getSimpleValueType();
9006 MVT ContainerVT = VT;
9007 if (VT.isFixedLengthVector())
9008 ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
9009
9010 // Recursively split concat_vectors with more than 2 operands:
9011 //
9012 // concat_vector op1, op2, op3, op4
9013 // ->
9014 // concat_vector (concat_vector op1, op2), (concat_vector op3, op4)
9015 //
9016 // This reduces the length of the chain of vslideups and allows us to
9017 // perform the vslideups at a smaller LMUL, limited to MF2.
9018 if (Op.getNumOperands() > 2 &&
9019 ContainerVT.bitsGE(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT))) {
9020 MVT HalfVT = VT.getHalfNumVectorElementsVT();
9021 assert(isPowerOf2_32(Op.getNumOperands()));
9022 size_t HalfNumOps = Op.getNumOperands() / 2;
9023 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
9024 Ops: Op->ops().take_front(N: HalfNumOps));
9025 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
9026 Ops: Op->ops().drop_front(N: HalfNumOps));
9027 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
9028 }
9029
9030 unsigned NumOpElts =
9031 Op.getOperand(i: 0).getSimpleValueType().getVectorMinNumElements();
9032 SDValue Vec = DAG.getUNDEF(VT);
9033 for (const auto &OpIdx : enumerate(First: Op->ops())) {
9034 SDValue SubVec = OpIdx.value();
9035 // Don't insert undef subvectors.
9036 if (SubVec.isUndef())
9037 continue;
9038 Vec = DAG.getInsertSubvector(DL, Vec, SubVec, Idx: OpIdx.index() * NumOpElts);
9039 }
9040 return Vec;
9041 }
9042 case ISD::LOAD: {
9043 auto *Load = cast<LoadSDNode>(Val&: Op);
9044 EVT VT = Load->getValueType(ResNo: 0);
9045 if (VT == MVT::f64) {
9046 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
9047 !Subtarget.is64Bit() && "Unexpected custom legalisation");
9048
9049 // Replace a double precision load with two i32 loads and a BuildPairF64.
9050 SDLoc DL(Op);
9051 SDValue BasePtr = Load->getBasePtr();
9052 SDValue Chain = Load->getChain();
9053
9054 SDValue Lo =
9055 DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo(),
9056 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
9057 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: 4));
9058 SDValue Hi = DAG.getLoad(
9059 VT: MVT::i32, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo().getWithOffset(O: 4),
9060 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
9061 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
9062 N2: Hi.getValue(R: 1));
9063
9064 // For big-endian, swap the order of Lo and Hi.
9065 if (!Subtarget.isLittleEndian())
9066 std::swap(a&: Lo, b&: Hi);
9067
9068 SDValue Pair = DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
9069 return DAG.getMergeValues(Ops: {Pair, Chain}, dl: DL);
9070 }
9071
9072 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9073 (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
9074 assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
9075
9076 // Determine the half-size type
9077 MVT HalfVT;
9078 if (VT == MVT::v2i32)
9079 HalfVT = MVT::i32;
9080 else if (VT == MVT::v4i16)
9081 HalfVT = MVT::v2i16;
9082 else // VT == MVT::v8i8
9083 HalfVT = MVT::v4i8;
9084
9085 SDLoc DL(Op);
9086 SDValue BasePtr = Load->getBasePtr();
9087 SDValue Chain = Load->getChain();
9088
9089 // Create two loads for the lower and upper halves
9090 SDValue Lo =
9091 DAG.getLoad(VT: HalfVT, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo(),
9092 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
9093 unsigned HalfSize = HalfVT.getStoreSize();
9094 BasePtr =
9095 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: HalfSize));
9096 SDValue Hi =
9097 DAG.getLoad(VT: HalfVT, dl: DL, Chain, Ptr: BasePtr,
9098 PtrInfo: Load->getPointerInfo().getWithOffset(O: HalfSize),
9099 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
9100 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
9101 N2: Hi.getValue(R: 1));
9102
9103 // Combine the two halves into the result vector
9104 SDValue Result;
9105 if (VT == MVT::v2i32) {
9106 // For v2i32, build vector from two i32 scalars
9107 Result = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
9108 } else {
9109 // For v4i16 and v8i8, use CONCAT_VECTORS
9110 Result = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
9111 }
9112
9113 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
9114 }
9115
9116 if (VT == MVT::bf16)
9117 return lowerXAndesBfHCvtBFloat16Load(Op, DAG);
9118
9119 // Handle normal vector tuple load.
9120 if (VT.isRISCVVectorTuple()) {
9121 SDLoc DL(Op);
9122 MVT XLenVT = Subtarget.getXLenVT();
9123 unsigned NF = VT.getRISCVVectorTupleNumFields();
9124 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
9125 unsigned NumElts = Sz / (NF * 8);
9126 int Log2LMUL = Log2_64(Value: NumElts) - 3;
9127
9128 auto Flag = SDNodeFlags();
9129 Flag.setNoUnsignedWrap(true);
9130 SDValue Ret = DAG.getUNDEF(VT);
9131 SDValue BasePtr = Load->getBasePtr();
9132 SDValue VROffset = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
9133 VROffset =
9134 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VROffset,
9135 N2: DAG.getConstant(Val: std::max(a: Log2LMUL, b: 0), DL, VT: XLenVT));
9136 SmallVector<SDValue, 8> OutChains;
9137
9138 // Load NF vector registers and combine them to a vector tuple.
9139 for (unsigned i = 0; i < NF; ++i) {
9140 SDValue LoadVal = DAG.getLoad(
9141 VT: MVT::getScalableVectorVT(VT: MVT::i8, NumElements: NumElts), dl: DL, Chain: Load->getChain(),
9142 Ptr: BasePtr, PtrInfo: MachinePointerInfo(Load->getAddressSpace()), Alignment: Align(8));
9143 OutChains.push_back(Elt: LoadVal.getValue(R: 1));
9144 Ret = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT, N1: Ret, N2: LoadVal,
9145 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
9146 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: BasePtr, N2: VROffset, Flags: Flag);
9147 }
9148 return DAG.getMergeValues(
9149 Ops: {Ret, DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains)}, dl: DL);
9150 }
9151
9152 if (auto V = expandUnalignedRVVLoad(Op, DAG))
9153 return V;
9154 if (Op.getValueType().isFixedLengthVector())
9155 return lowerFixedLengthVectorLoadToRVV(Op, DAG);
9156 return Op;
9157 }
9158 case ISD::STORE: {
9159 auto *Store = cast<StoreSDNode>(Val&: Op);
9160 SDValue StoredVal = Store->getValue();
9161 EVT VT = StoredVal.getValueType();
9162
9163 if (VT == MVT::f64) {
9164 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
9165 !Subtarget.is64Bit() && "Unexpected custom legalisation");
9166
9167 // Replace a double precision store with a SplitF64 and i32 stores.
9168 SDValue DL(Op);
9169 SDValue BasePtr = Store->getBasePtr();
9170 SDValue Chain = Store->getChain();
9171 SDValue Split = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
9172 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: StoredVal);
9173
9174 SDValue Lo = Split.getValue(R: 0);
9175 SDValue Hi = Split.getValue(R: 1);
9176
9177 // For big-endian, swap the order of Lo and Hi before storing.
9178 if (!Subtarget.isLittleEndian())
9179 std::swap(a&: Lo, b&: Hi);
9180
9181 SDValue LoStore = DAG.getStore(
9182 Chain, dl: DL, Val: Lo, Ptr: BasePtr, PtrInfo: Store->getPointerInfo(),
9183 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9184 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: 4));
9185 SDValue HiStore = DAG.getStore(
9186 Chain, dl: DL, Val: Hi, Ptr: BasePtr, PtrInfo: Store->getPointerInfo().getWithOffset(O: 4),
9187 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9188 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: LoStore, N2: HiStore);
9189 }
9190 if (VT == MVT::i64) {
9191 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
9192 "Unexpected custom legalisation");
9193 if (Store->isTruncatingStore())
9194 return SDValue();
9195
9196 if (Store->getAlign() < Subtarget.getZilsdAlign())
9197 return SDValue();
9198
9199 SDLoc DL(Op);
9200 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i32, N1: StoredVal,
9201 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
9202 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i32, N1: StoredVal,
9203 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
9204
9205 return DAG.getMemIntrinsicNode(
9206 Opcode: RISCVISD::SD_RV32, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
9207 Ops: {Store->getChain(), Lo, Hi, Store->getBasePtr()}, MemVT: MVT::i64,
9208 MMO: Store->getMemOperand());
9209 }
9210
9211 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9212 (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
9213 assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
9214
9215 auto *Store = cast<StoreSDNode>(Val&: Op);
9216 SDValue Val = Store->getValue();
9217
9218 // Determine the half-size type
9219 MVT HalfVT;
9220 if (VT == MVT::v2i32)
9221 HalfVT = MVT::i32;
9222 else if (VT == MVT::v4i16)
9223 HalfVT = MVT::v2i16;
9224 else // VT == MVT::v8i8
9225 HalfVT = MVT::v4i8;
9226
9227 SDLoc DL(Op);
9228 SDValue BasePtr = Store->getBasePtr();
9229 SDValue Chain = Store->getChain();
9230
9231 // Extract the two halves from the vector
9232 SDValue Lo, Hi;
9233 if (VT == MVT::v2i32) {
9234 // For v2i32, extract two i32 scalars
9235 Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Val,
9236 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9237 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Val,
9238 N2: DAG.getVectorIdxConstant(Val: 1, DL));
9239 } else {
9240 // For v4i16 and v8i8, extract two vector halves
9241 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HalfVT, N1: Val,
9242 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9243 unsigned HalfNumElts = HalfVT.getVectorNumElements();
9244 Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HalfVT, N1: Val,
9245 N2: DAG.getVectorIdxConstant(Val: HalfNumElts, DL));
9246 }
9247
9248 // Create two stores for the lower and upper halves
9249 SDValue LoStore = DAG.getStore(
9250 Chain, dl: DL, Val: Lo, Ptr: BasePtr, PtrInfo: Store->getPointerInfo(),
9251 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9252 unsigned HalfSize = HalfVT.getStoreSize();
9253 BasePtr =
9254 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: HalfSize));
9255 SDValue HiStore = DAG.getStore(
9256 Chain, dl: DL, Val: Hi, Ptr: BasePtr,
9257 PtrInfo: Store->getPointerInfo().getWithOffset(O: HalfSize),
9258 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9259
9260 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: LoStore, N2: HiStore);
9261 }
9262
9263 if (VT == MVT::bf16)
9264 return lowerXAndesBfHCvtBFloat16Store(Op, DAG);
9265
9266 // Handle normal vector tuple store.
9267 if (VT.isRISCVVectorTuple()) {
9268 SDLoc DL(Op);
9269 MVT XLenVT = Subtarget.getXLenVT();
9270 unsigned NF = VT.getRISCVVectorTupleNumFields();
9271 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
9272 unsigned NumElts = Sz / (NF * 8);
9273 int Log2LMUL = Log2_64(Value: NumElts) - 3;
9274
9275 auto Flag = SDNodeFlags();
9276 Flag.setNoUnsignedWrap(true);
9277 SDValue Ret;
9278 SDValue Chain = Store->getChain();
9279 SDValue BasePtr = Store->getBasePtr();
9280 SDValue VROffset = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
9281 VROffset =
9282 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VROffset,
9283 N2: DAG.getConstant(Val: std::max(a: Log2LMUL, b: 0), DL, VT: XLenVT));
9284
9285 // Extract subregisters in a vector tuple and store them individually.
9286 for (unsigned i = 0; i < NF; ++i) {
9287 auto Extract =
9288 DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL,
9289 VT: MVT::getScalableVectorVT(VT: MVT::i8, NumElements: NumElts), N1: StoredVal,
9290 N2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
9291 Ret = DAG.getStore(Chain, dl: DL, Val: Extract, Ptr: BasePtr,
9292 PtrInfo: MachinePointerInfo(Store->getAddressSpace()),
9293 Alignment: Store->getBaseAlign(),
9294 MMOFlags: Store->getMemOperand()->getFlags());
9295 Chain = Ret.getValue(R: 0);
9296 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: BasePtr, N2: VROffset, Flags: Flag);
9297 }
9298 return Ret;
9299 }
9300
9301 if (auto V = expandUnalignedRVVStore(Op, DAG))
9302 return V;
9303 if (Op.getOperand(i: 1).getValueType().isFixedLengthVector())
9304 return lowerFixedLengthVectorStoreToRVV(Op, DAG);
9305 return Op;
9306 }
9307 case ISD::VP_LOAD:
9308 if (SDValue V = expandUnalignedVPLoad(Op, DAG))
9309 return V;
9310 [[fallthrough]];
9311 case ISD::MLOAD:
9312 return lowerMaskedLoad(Op, DAG);
9313 case ISD::VP_LOAD_FF:
9314 return lowerLoadFF(Op, DAG);
9315 case ISD::VP_STORE:
9316 if (SDValue V = expandUnalignedVPStore(Op, DAG))
9317 return V;
9318 [[fallthrough]];
9319 case ISD::MSTORE:
9320 return lowerMaskedStore(Op, DAG);
9321 case ISD::VECTOR_COMPRESS:
9322 return lowerVectorCompress(Op, DAG);
9323 case ISD::SELECT_CC: {
9324 // This occurs because we custom legalize SETGT and SETUGT for setcc. That
9325 // causes LegalizeDAG to think we need to custom legalize select_cc. Expand
9326 // into separate SETCC+SELECT just like LegalizeDAG.
9327 SDValue Tmp1 = Op.getOperand(i: 0);
9328 SDValue Tmp2 = Op.getOperand(i: 1);
9329 SDValue True = Op.getOperand(i: 2);
9330 SDValue False = Op.getOperand(i: 3);
9331 EVT VT = Op.getValueType();
9332 SDValue CC = Op.getOperand(i: 4);
9333 EVT CmpVT = Tmp1.getValueType();
9334 EVT CCVT =
9335 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: CmpVT);
9336 SDLoc DL(Op);
9337 SDValue Cond =
9338 DAG.getNode(Opcode: ISD::SETCC, DL, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Op->getFlags());
9339 return DAG.getSelect(DL, VT, Cond, LHS: True, RHS: False);
9340 }
9341 case ISD::SETCC: {
9342 MVT OpVT = Op.getOperand(i: 0).getSimpleValueType();
9343 if (OpVT.isScalarInteger()) {
9344 MVT VT = Op.getSimpleValueType();
9345 SDValue LHS = Op.getOperand(i: 0);
9346 SDValue RHS = Op.getOperand(i: 1);
9347 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9348 assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) &&
9349 "Unexpected CondCode");
9350
9351 SDLoc DL(Op);
9352
9353 // If the RHS is a constant in the range [-2049, 0) or (0, 2046], we can
9354 // convert this to the equivalent of (set(u)ge X, C+1) by using
9355 // (xori (slti(u) X, C+1), 1). This avoids materializing a small constant
9356 // in a register.
9357 if (isa<ConstantSDNode>(Val: RHS)) {
9358 int64_t Imm = cast<ConstantSDNode>(Val&: RHS)->getSExtValue();
9359 if (Imm != 0 && isInt<12>(x: (uint64_t)Imm + 1)) {
9360 // If this is an unsigned compare and the constant is -1, incrementing
9361 // the constant would change behavior. The result should be false.
9362 if (CCVal == ISD::SETUGT && Imm == -1)
9363 return DAG.getConstant(Val: 0, DL, VT);
9364 // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
9365 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
9366 SDValue SetCC = DAG.getSetCC(
9367 DL, VT, LHS, RHS: DAG.getSignedConstant(Val: Imm + 1, DL, VT: OpVT), Cond: CCVal);
9368 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
9369 }
9370 // Lower (setugt X, 2047) as (setne (srl X, 11), 0).
9371 if (CCVal == ISD::SETUGT && Imm == 2047) {
9372 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: OpVT, N1: LHS,
9373 N2: DAG.getShiftAmountConstant(Val: 11, VT: OpVT, DL));
9374 return DAG.getSetCC(DL, VT, LHS: Shift, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT),
9375 Cond: ISD::SETNE);
9376 }
9377 }
9378
9379 // Not a constant we could handle, swap the operands and condition code to
9380 // SETLT/SETULT.
9381 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
9382 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: CCVal);
9383 }
9384
9385 MVT VT = Op.getSimpleValueType();
9386 if (Subtarget.hasStdExtP() && VT.isFixedLengthVector()) {
9387 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9388 SDValue LHS = Op.getOperand(i: 0);
9389 SDValue RHS = Op.getOperand(i: 1);
9390 SDLoc DL(Op);
9391 if (CCVal == ISD::SETNE) {
9392 // Convert setne X, 0 to setult 0, X.
9393 if (ISD::isConstantSplatVectorAllZeros(N: RHS.getNode())) {
9394 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: ISD::SETULT);
9395 }
9396
9397 // Not a constant we could handle, convert to SETEQ+Invert
9398 SDValue SetCC = DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::SETEQ);
9399 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
9400 }
9401
9402 if (CCVal == ISD::SETGT) {
9403 if (ISD::isConstantSplatVectorAllOnes(N: RHS.getNode())) {
9404 SDValue SetCC =
9405 DAG.getSetCC(DL, VT, LHS, RHS: DAG.getConstant(Val: 0, DL, VT), Cond: ISD::SETLT);
9406 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
9407 }
9408
9409 // Not a constant we could handle, swap the operands and condition code
9410 // to SETLT.
9411 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
9412 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: CCVal);
9413 }
9414
9415 return SDValue();
9416 }
9417
9418 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 0), Subtarget, TLI: *this))
9419 return SplitVectorOp(Op, DAG);
9420
9421 return lowerToScalableOp(Op, DAG);
9422 }
9423 case ISD::ADD:
9424 case ISD::SUB:
9425 case ISD::SDIV:
9426 case ISD::SREM:
9427 case ISD::UDIV:
9428 case ISD::UREM:
9429 case ISD::BSWAP:
9430 case ISD::CTPOP:
9431 return lowerToScalableOp(Op, DAG);
9432 case ISD::VSELECT: {
9433 EVT VT = Op.getValueType();
9434 // Split 64-bit vector VSELECT on RV32 with P extension for v4i16 and v8i8
9435 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9436 (VT == MVT::v4i16 || VT == MVT::v8i8)) {
9437 SDLoc DL(Op);
9438 SDValue Mask = Op.getOperand(i: 0);
9439 SDValue TrueVal = Op.getOperand(i: 1);
9440 SDValue FalseVal = Op.getOperand(i: 2);
9441
9442 // Split all three operands into two halves
9443 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Mask, DL);
9444 auto [TrueLo, TrueHi] = DAG.SplitVector(N: TrueVal, DL);
9445 auto [FalseLo, FalseHi] = DAG.SplitVector(N: FalseVal, DL);
9446
9447 // Perform VSELECT on each half
9448 SDValue ResLo = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: TrueLo.getValueType(),
9449 N1: MaskLo, N2: TrueLo, N3: FalseLo);
9450 SDValue ResHi = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: TrueHi.getValueType(),
9451 N1: MaskHi, N2: TrueHi, N3: FalseHi);
9452
9453 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: ResLo, N2: ResHi);
9454 }
9455 return lowerToScalableOp(Op, DAG);
9456 }
9457 case ISD::AND:
9458 case ISD::OR:
9459 case ISD::XOR:
9460 case ISD::MUL:
9461 case ISD::MULHS:
9462 case ISD::MULHU: {
9463 EVT VT = Op.getValueType();
9464 unsigned Opc = Op.getOpcode();
9465 // Split 64-bit vector AND/OR/XOR/MUL/MULHS/MULHU on RV32 with P extension
9466 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9467 (VT == MVT::v4i16 || VT == MVT::v8i8)) {
9468 SDLoc DL(Op);
9469 SDValue LHS = Op.getOperand(i: 0);
9470 SDValue RHS = Op.getOperand(i: 1);
9471
9472 // Determine the half-size type
9473 MVT HalfVT = (VT == MVT::v4i16) ? MVT::v2i16 : MVT::v4i8;
9474
9475 // Extract the two halves from LHS
9476 auto [LHSLo, LHSHi] = DAG.SplitVector(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
9477
9478 // Extract the two halves from RHS
9479 auto [RHSLo, RHSHi] = DAG.SplitVector(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
9480
9481 // Perform the operation on each half
9482 SDValue ResLo = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: LHSLo, N2: RHSLo);
9483 SDValue ResHi = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: LHSHi, N2: RHSHi);
9484
9485 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: ResLo, N2: ResHi);
9486 }
9487 // Lower v4i8/v2i16 MUL/MULHS/MULHU via widening multiply + srl + truncate.
9488 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9489 (Opc == ISD::MUL || Opc == ISD::MULHS || Opc == ISD::MULHU) &&
9490 (VT == MVT::v4i8 || VT == MVT::v2i16)) {
9491 assert((VT == MVT::v4i8 || Opc == ISD::MUL) &&
9492 "Unexpected custom legalisation");
9493 SDLoc DL(Op);
9494 MVT WideVT = (VT == MVT::v4i8) ? MVT::v4i16 : MVT::v2i32;
9495 unsigned WMulOpc =
9496 (Opc == ISD::MULHU) ? RISCVISD::PWMULU : RISCVISD::PWMUL;
9497 SDValue Res =
9498 DAG.getNode(Opcode: WMulOpc, DL, VT: WideVT, N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
9499 if (Opc != ISD::MUL) {
9500 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
9501 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: WideVT, N1: Res,
9502 N2: DAG.getConstant(Val: EltBits, DL, VT: WideVT));
9503 }
9504 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
9505 }
9506 // Lower v8i8 MULHS/MULHU on RV64 via a pair of widening byte multiplies
9507 // (picking out the even/odd result lanes) recombined with PPAIRO.
9508 if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::v8i8 &&
9509 Opc != ISD::MUL) {
9510 SDLoc DL(Op);
9511 SDValue LHS = Op.getOperand(i: 0);
9512 SDValue RHS = Op.getOperand(i: 1);
9513 unsigned LoOpc = Opc == ISD::MULHU ? RISCVISD::PMULU_HALVES_00
9514 : RISCVISD::PMUL_HALVES_00;
9515 unsigned HiOpc = Opc == ISD::MULHU ? RISCVISD::PMULU_HALVES_11
9516 : RISCVISD::PMUL_HALVES_11;
9517 SDValue Lo = DAG.getNode(Opcode: LoOpc, DL, VT: MVT::v4i16, N1: LHS, N2: RHS);
9518 SDValue Hi = DAG.getNode(Opcode: HiOpc, DL, VT: MVT::v4i16, N1: LHS, N2: RHS);
9519 return DAG.getNode(Opcode: RISCVISD::PPAIRO, DL, VT, N1: DAG.getBitcast(VT, V: Lo),
9520 N2: DAG.getBitcast(VT, V: Hi));
9521 }
9522 return lowerToScalableOp(Op, DAG);
9523 }
9524 case ISD::ANY_EXTEND_VECTOR_INREG: {
9525 EVT VT = Op.getValueType();
9526 assert(Subtarget.hasStdExtP() && Subtarget.is64Bit() &&
9527 (VT == MVT::v2i32 || VT == MVT::v4i16) &&
9528 "Unexpected custom legalisation");
9529 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL: SDLoc(Op), VT,
9530 Operand: Op.getOperand(i: 0));
9531 }
9532 case ISD::SHL:
9533 case ISD::SRL:
9534 case ISD::SRA:
9535 case ISD::SSHLSAT:
9536 if (Op.getSimpleValueType().isFixedLengthVector()) {
9537 if (Subtarget.hasStdExtP()) {
9538 SDValue ShAmtVec = Op.getOperand(i: 1);
9539 SDValue SplatVal;
9540 if (ShAmtVec.getOpcode() == ISD::SPLAT_VECTOR)
9541 SplatVal = ShAmtVec.getOperand(i: 0);
9542 else if (ShAmtVec.getOpcode() == ISD::BUILD_VECTOR)
9543 SplatVal = cast<BuildVectorSDNode>(Val&: ShAmtVec)->getSplatValue();
9544
9545 if (!SplatVal)
9546 return SDValue();
9547
9548 unsigned Opc;
9549 switch (Op.getOpcode()) {
9550 default:
9551 llvm_unreachable("Unexpected opcode");
9552 case ISD::SHL:
9553 Opc = RISCVISD::PSHL;
9554 break;
9555 case ISD::SRL:
9556 Opc = RISCVISD::PSRL;
9557 break;
9558 case ISD::SRA:
9559 Opc = RISCVISD::PSRA;
9560 break;
9561 case ISD::SSHLSAT:
9562 Opc = RISCVISD::PSSHA;
9563 break;
9564 }
9565 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: Op.getValueType(), N1: Op.getOperand(i: 0),
9566 N2: SplatVal);
9567 }
9568 return lowerToScalableOp(Op, DAG);
9569 }
9570 assert(Op.getOpcode() != ISD::SSHLSAT);
9571 // This can be called for an i32 shift amount that needs to be promoted.
9572 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
9573 "Unexpected custom legalisation");
9574 return SDValue();
9575 case ISD::MASKED_UDIV:
9576 case ISD::MASKED_SDIV:
9577 case ISD::MASKED_UREM:
9578 case ISD::MASKED_SREM: {
9579 SDLoc DL(Op);
9580 MVT VT = Op.getSimpleValueType();
9581 MVT ContainerVT = getContainerForFixedLengthVector(VT);
9582 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
9583 SDValue Res = DAG.getNode(
9584 Opcode: getRISCVVLOp(Op), DL, VT: ContainerVT,
9585 N1: convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i: 0), DAG, Subtarget),
9586 N2: convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i: 1), DAG, Subtarget),
9587 N3: DAG.getUNDEF(VT: ContainerVT),
9588 N4: convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Op.getOperand(i: 2),
9589 DAG, Subtarget),
9590 N5: VL);
9591 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
9592 }
9593 case ISD::FABS:
9594 case ISD::FNEG:
9595 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9596 return lowerFABSorFNEG(Op, DAG, Subtarget);
9597 [[fallthrough]];
9598 case ISD::FADD:
9599 case ISD::FSUB:
9600 case ISD::FMUL:
9601 case ISD::FDIV:
9602 case ISD::FSQRT:
9603 case ISD::FMA:
9604 case ISD::FMINNUM:
9605 case ISD::FMAXNUM:
9606 case ISD::FMINIMUMNUM:
9607 case ISD::FMAXIMUMNUM:
9608 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
9609 return SplitVectorOp(Op, DAG);
9610 [[fallthrough]];
9611 case ISD::AVGFLOORS:
9612 case ISD::AVGFLOORU:
9613 case ISD::AVGCEILS:
9614 case ISD::AVGCEILU:
9615 case ISD::SMIN:
9616 case ISD::SMAX:
9617 case ISD::UMIN:
9618 case ISD::UMAX:
9619 case ISD::UADDSAT:
9620 case ISD::USUBSAT:
9621 case ISD::SADDSAT:
9622 case ISD::SSUBSAT:
9623 return lowerToScalableOp(Op, DAG);
9624 case ISD::ABDS:
9625 case ISD::ABDU: {
9626 EVT VT = Op->getValueType(ResNo: 0);
9627 if (Subtarget.hasStdExtZvabd() && VT.isVector())
9628 return lowerToScalableOp(Op, DAG);
9629
9630 SDLoc dl(Op);
9631 SDValue LHS = DAG.getFreeze(V: Op->getOperand(Num: 0));
9632 SDValue RHS = DAG.getFreeze(V: Op->getOperand(Num: 1));
9633 bool IsSigned = Op->getOpcode() == ISD::ABDS;
9634
9635 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
9636 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
9637 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
9638 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
9639 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
9640 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
9641 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
9642 }
9643 case ISD::ABS:
9644 case ISD::ABS_MIN_POISON:
9645 return lowerABS(Op, DAG);
9646 case ISD::CTLZ:
9647 case ISD::CTLZ_ZERO_POISON:
9648 case ISD::CTTZ:
9649 case ISD::CTTZ_ZERO_POISON:
9650 if (Subtarget.hasStdExtZvbb())
9651 return lowerToScalableOp(Op, DAG);
9652 assert(Op.getOpcode() != ISD::CTTZ);
9653 return lowerCTLZ_CTTZ_ZERO_POISON(Op, DAG);
9654 case ISD::CLMUL:
9655 case ISD::CLMULH: {
9656 SDLoc DL(Op);
9657 MVT VT = Op.getSimpleValueType();
9658 MVT XLenVT = Subtarget.getXLenVT();
9659
9660 if (!VT.isVector()) {
9661 // The op needs to be performed in a vector register.
9662 assert(Subtarget.hasStdExtZvbc() && Subtarget.is64Bit() && VT == XLenVT &&
9663 "Unexpected custom legalisation");
9664 // We can't implicitly use vector registers with noimplicitfloat.
9665 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
9666 Kind: Attribute::NoImplicitFloat))
9667 return SDValue();
9668 MVT VecVT = MVT::getScalableVectorVT(VT, NumElements: 1);
9669 SDValue VL = DAG.getConstant(Val: 1, DL, VT: XLenVT);
9670 SDValue Op0 =
9671 lowerScalarInsert(Scalar: Op.getOperand(i: 0), VL, VT: VecVT, DL, DAG, Subtarget);
9672 SDValue Op1 =
9673 lowerScalarInsert(Scalar: Op.getOperand(i: 1), VL, VT: VecVT, DL, DAG, Subtarget);
9674 SDValue Res = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: VecVT, N1: Op0, N2: Op1);
9675 return DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Res);
9676 }
9677
9678 // If the scalable vector version of this op is Legal, convert to scalable.
9679 if (VT.isFixedLengthVector() &&
9680 (VT.getVectorElementType() == MVT::i64 || Subtarget.hasStdExtZvbc32e()))
9681 return lowerToScalableOp(Op, DAG);
9682
9683 assert(Op.getOpcode() == ISD::CLMUL && Subtarget.hasStdExtZvbc() &&
9684 "Unexpected custom legalisation");
9685 // Promote to i64 vector.
9686 MVT I64VecVT = VT.changeVectorElementType(EltVT: MVT::i64);
9687 SDValue Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: I64VecVT, Operand: Op.getOperand(i: 0));
9688 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: I64VecVT, Operand: Op.getOperand(i: 1));
9689 SDValue CLMUL = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: I64VecVT, N1: Op0, N2: Op1);
9690 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CLMUL);
9691 }
9692 case ISD::FCOPYSIGN:
9693 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9694 return lowerFCOPYSIGN(Op, DAG, Subtarget);
9695 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
9696 return SplitVectorOp(Op, DAG);
9697 return lowerToScalableOp(Op, DAG);
9698 case ISD::STRICT_FADD:
9699 case ISD::STRICT_FSUB:
9700 case ISD::STRICT_FMUL:
9701 case ISD::STRICT_FDIV:
9702 case ISD::STRICT_FSQRT:
9703 case ISD::STRICT_FMA:
9704 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
9705 return SplitStrictFPVectorOp(Op, DAG);
9706 return lowerToScalableOp(Op, DAG);
9707 case ISD::STRICT_FSETCC:
9708 case ISD::STRICT_FSETCCS:
9709 return lowerVectorStrictFSetcc(Op, DAG);
9710 case ISD::STRICT_FCEIL:
9711 case ISD::STRICT_FRINT:
9712 case ISD::STRICT_FFLOOR:
9713 case ISD::STRICT_FTRUNC:
9714 case ISD::STRICT_FNEARBYINT:
9715 case ISD::STRICT_FROUND:
9716 case ISD::STRICT_FROUNDEVEN:
9717 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
9718 case ISD::MGATHER:
9719 case ISD::VP_GATHER:
9720 return lowerMaskedGather(Op, DAG);
9721 case ISD::MSCATTER:
9722 case ISD::VP_SCATTER:
9723 return lowerMaskedScatter(Op, DAG);
9724 case ISD::GET_ROUNDING:
9725 return lowerGET_ROUNDING(Op, DAG);
9726 case ISD::SET_ROUNDING:
9727 return lowerSET_ROUNDING(Op, DAG);
9728 case ISD::GET_FPENV:
9729 return lowerGET_FPENV(Op, DAG);
9730 case ISD::SET_FPENV:
9731 return lowerSET_FPENV(Op, DAG);
9732 case ISD::RESET_FPENV:
9733 return lowerRESET_FPENV(Op, DAG);
9734 case ISD::GET_FPMODE:
9735 return lowerGET_FPMODE(Op, DAG);
9736 case ISD::SET_FPMODE:
9737 return lowerSET_FPMODE(Op, DAG);
9738 case ISD::RESET_FPMODE:
9739 return lowerRESET_FPMODE(Op, DAG);
9740 case ISD::EH_DWARF_CFA:
9741 return lowerEH_DWARF_CFA(Op, DAG);
9742 case ISD::VP_MERGE:
9743 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
9744 return lowerVPMergeMask(Op, DAG);
9745 [[fallthrough]];
9746 case ISD::VP_SDIV:
9747 case ISD::VP_UDIV:
9748 case ISD::VP_SREM:
9749 case ISD::VP_UREM:
9750 return lowerVPOp(Op, DAG);
9751 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9752 return lowerVPStridedLoad(Op, DAG);
9753 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9754 return lowerVPStridedStore(Op, DAG);
9755 case ISD::EXPERIMENTAL_VP_SPLICE:
9756 return lowerVPSpliceExperimental(Op, DAG);
9757 case ISD::EXPERIMENTAL_VP_REVERSE:
9758 return lowerVPReverseExperimental(Op, DAG);
9759 case ISD::CLEAR_CACHE: {
9760 assert(getTargetMachine().getTargetTriple().isOSLinux() &&
9761 "llvm.clear_cache only needs custom lower on Linux targets");
9762 SDLoc DL(Op);
9763 SDValue Flags = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
9764 return emitFlushICache(DAG, InChain: Op.getOperand(i: 0), Start: Op.getOperand(i: 1),
9765 End: Op.getOperand(i: 2), Flags, DL);
9766 }
9767 case ISD::DYNAMIC_STACKALLOC:
9768 return lowerDYNAMIC_STACKALLOC(Op, DAG);
9769 case ISD::INIT_TRAMPOLINE:
9770 return lowerINIT_TRAMPOLINE(Op, DAG);
9771 case ISD::ADJUST_TRAMPOLINE:
9772 return lowerADJUST_TRAMPOLINE(Op, DAG);
9773 case ISD::PARTIAL_REDUCE_UMLA:
9774 case ISD::PARTIAL_REDUCE_SMLA:
9775 case ISD::PARTIAL_REDUCE_SUMLA:
9776 return lowerPARTIAL_REDUCE_MLA(Op, DAG);
9777 case ISD::CTTZ_ELTS:
9778 case ISD::CTTZ_ELTS_ZERO_POISON:
9779 return lowerCttzElts(Op, DAG, Subtarget);
9780 case ISD::CONVERT_FROM_ARBITRARY_FP:
9781 return lowerCONVERT_FROM_ARBITRARY_FP(Op, DAG, Subtarget);
9782 }
9783}
9784
9785SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
9786 SDValue Start, SDValue End,
9787 SDValue Flags, SDLoc DL) const {
9788 MakeLibCallOptions CallOptions;
9789 std::pair<SDValue, SDValue> CallResult =
9790 makeLibCall(DAG, LC: RTLIB::RISCV_FLUSH_ICACHE, RetVT: MVT::isVoid,
9791 Ops: {Start, End, Flags}, CallOptions, dl: DL, Chain: InChain);
9792
9793 // This function returns void so only the out chain matters.
9794 return CallResult.second;
9795}
9796
9797SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
9798 SelectionDAG &DAG) const {
9799 if (!Subtarget.is64Bit())
9800 llvm::reportFatalUsageError(reason: "Trampolines only implemented for RV64");
9801
9802 // Create an MCCodeEmitter to encode instructions.
9803 TargetLoweringObjectFile *TLO = getTargetMachine().getObjFileLowering();
9804 assert(TLO);
9805 MCContext &MCCtx = TLO->getContext();
9806
9807 std::unique_ptr<MCCodeEmitter> CodeEmitter(
9808 createRISCVMCCodeEmitter(MCII: *getTargetMachine().getMCInstrInfo(), Ctx&: MCCtx));
9809
9810 SDValue Root = Op.getOperand(i: 0);
9811 SDValue Trmp = Op.getOperand(i: 1); // trampoline
9812 SDLoc dl(Op);
9813
9814 const Value *TrmpAddr = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
9815
9816 // We store in the trampoline buffer the following instructions and data.
9817 // Offset:
9818 // 0: auipc t2, 0
9819 // 4: ld t0, 24(t2)
9820 // 8: ld t2, 16(t2)
9821 // 12: jalr t0
9822 // 16: <StaticChainOffset>
9823 // 24: <FunctionAddressOffset>
9824 // 32:
9825 // Offset with branch control flow protection enabled:
9826 // 0: lpad <imm20>
9827 // 4: auipc t3, 0
9828 // 8: ld t2, 28(t3)
9829 // 12: ld t3, 20(t3)
9830 // 16: jalr t2
9831 // 20: <StaticChainOffset>
9832 // 28: <FunctionAddressOffset>
9833 // 36:
9834
9835 const MachineFunction &MF = DAG.getMachineFunction();
9836 const bool HasCFBranch =
9837 MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
9838 const unsigned StaticChainIdx = HasCFBranch ? 5 : 4;
9839 const unsigned StaticChainOffset = StaticChainIdx * 4;
9840 const unsigned FunctionAddressOffset = StaticChainOffset + 8;
9841
9842 const MCSubtargetInfo &STI = getTargetMachine().getMCSubtargetInfo();
9843 auto GetEncoding = [&](const MCInst &MC) {
9844 SmallVector<char, 4> CB;
9845 SmallVector<MCFixup> Fixups;
9846 CodeEmitter->encodeInstruction(Inst: MC, CB, Fixups, STI);
9847 uint32_t Encoding = support::endian::read32le(P: CB.data());
9848 return Encoding;
9849 };
9850
9851 SmallVector<SDValue> OutChains;
9852
9853 SmallVector<uint32_t> Encodings;
9854 if (!HasCFBranch) {
9855 Encodings.append(
9856 IL: {// auipc t2, 0
9857 // Loads the current PC into t2.
9858 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X7).addImm(Val: 0)),
9859 // ld t0, 24(t2)
9860 // Loads the function address into t0. Note that we are using offsets
9861 // pc-relative to the first instruction of the trampoline.
9862 GetEncoding(MCInstBuilder(RISCV::LD)
9863 .addReg(Reg: RISCV::X5)
9864 .addReg(Reg: RISCV::X7)
9865 .addImm(Val: FunctionAddressOffset)),
9866 // ld t2, 16(t2)
9867 // Load the value of the static chain.
9868 GetEncoding(MCInstBuilder(RISCV::LD)
9869 .addReg(Reg: RISCV::X7)
9870 .addReg(Reg: RISCV::X7)
9871 .addImm(Val: StaticChainOffset)),
9872 // jalr t0
9873 // Jump to the function.
9874 GetEncoding(MCInstBuilder(RISCV::JALR)
9875 .addReg(Reg: RISCV::X0)
9876 .addReg(Reg: RISCV::X5)
9877 .addImm(Val: 0))});
9878 } else {
9879 Encodings.append(
9880 IL: {// auipc x0, <imm20> (lpad <imm20>)
9881 // Landing pad.
9882 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X0).addImm(Val: 0)),
9883 // auipc t3, 0
9884 // Loads the current PC into t3.
9885 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X28).addImm(Val: 0)),
9886 // ld t2, (FunctionAddressOffset - 4)(t3)
9887 // Loads the function address into t2. Note that we are using offsets
9888 // pc-relative to the SECOND instruction of the trampoline.
9889 GetEncoding(MCInstBuilder(RISCV::LD)
9890 .addReg(Reg: RISCV::X7)
9891 .addReg(Reg: RISCV::X28)
9892 .addImm(Val: FunctionAddressOffset - 4)),
9893 // ld t3, (StaticChainOffset - 4)(t3)
9894 // Load the value of the static chain.
9895 GetEncoding(MCInstBuilder(RISCV::LD)
9896 .addReg(Reg: RISCV::X28)
9897 .addReg(Reg: RISCV::X28)
9898 .addImm(Val: StaticChainOffset - 4)),
9899 // jalr t2
9900 // Software-guarded jump to the function.
9901 GetEncoding(MCInstBuilder(RISCV::JALR)
9902 .addReg(Reg: RISCV::X0)
9903 .addReg(Reg: RISCV::X7)
9904 .addImm(Val: 0))});
9905 }
9906
9907 // Store encoded instructions.
9908 for (auto [Idx, Encoding] : llvm::enumerate(First&: Encodings)) {
9909 SDValue Addr = Idx > 0 ? DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Trmp,
9910 N2: DAG.getConstant(Val: Idx * 4, DL: dl, VT: MVT::i64))
9911 : Trmp;
9912 OutChains.push_back(Elt: DAG.getTruncStore(
9913 Chain: Root, dl, Val: DAG.getConstant(Val: Encoding, DL: dl, VT: MVT::i64), Ptr: Addr,
9914 PtrInfo: MachinePointerInfo(TrmpAddr, Idx * 4), SVT: MVT::i32));
9915 }
9916
9917 // Now store the variable part of the trampoline.
9918 SDValue FunctionAddress = Op.getOperand(i: 2);
9919 SDValue StaticChain = Op.getOperand(i: 3);
9920
9921 // Store the given static chain and function pointer in the trampoline buffer.
9922 struct OffsetValuePair {
9923 const unsigned Offset;
9924 const SDValue Value;
9925 SDValue Addr = SDValue(); // Used to cache the address.
9926 } OffsetValues[] = {
9927 {.Offset: StaticChainOffset, .Value: StaticChain},
9928 {.Offset: FunctionAddressOffset, .Value: FunctionAddress},
9929 };
9930 for (auto &OffsetValue : OffsetValues) {
9931 SDValue Addr =
9932 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Trmp,
9933 N2: DAG.getConstant(Val: OffsetValue.Offset, DL: dl, VT: MVT::i64));
9934 OffsetValue.Addr = Addr;
9935 OutChains.push_back(
9936 Elt: DAG.getStore(Chain: Root, dl, Val: OffsetValue.Value, Ptr: Addr,
9937 PtrInfo: MachinePointerInfo(TrmpAddr, OffsetValue.Offset)));
9938 }
9939
9940 assert(OutChains.size() == StaticChainIdx + 2 &&
9941 "Size of OutChains mismatch");
9942 SDValue StoreToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9943
9944 // The end of instructions of trampoline is the same as the static chain
9945 // address that we computed earlier.
9946 SDValue EndOfTrmp = OffsetValues[0].Addr;
9947
9948 // Call clear cache on the trampoline instructions.
9949 SDValue Chain = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: dl, VT: MVT::Other, N1: StoreToken,
9950 N2: Trmp, N3: EndOfTrmp);
9951
9952 return Chain;
9953}
9954
9955SDValue RISCVTargetLowering::lowerADJUST_TRAMPOLINE(SDValue Op,
9956 SelectionDAG &DAG) const {
9957 if (!Subtarget.is64Bit())
9958 llvm::reportFatalUsageError(reason: "Trampolines only implemented for RV64");
9959
9960 return Op.getOperand(i: 0);
9961}
9962
9963SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
9964 SelectionDAG &DAG) const {
9965 // Currently, only the vdot4a and vdot4au case (from zvdot4a8i) should be
9966 // legal.
9967 // TODO: There are many other sub-cases we could potentially lower, are
9968 // any of them worthwhile? Ex: via vredsum, vwredsum, vwwmaccu, etc..
9969 SDLoc DL(Op);
9970 MVT VT = Op.getSimpleValueType();
9971 SDValue Accum = Op.getOperand(i: 0);
9972 SDValue A = Op.getOperand(i: 1);
9973 SDValue B = Op.getOperand(i: 2);
9974 MVT ArgVT = A.getSimpleValueType();
9975 assert(ArgVT == B.getSimpleValueType() &&
9976 ArgVT.getVectorElementType() == MVT::i8);
9977 (void)ArgVT;
9978
9979 // vdot4a* only produces an i32 result. For an i64 accumulator, perform the
9980 // dot product into a fresh i32 accumulator (each result is the sum of four
9981 // i8 products, which cannot overflow i32), reduce those i32 partial sums
9982 // down to the accumulator's element count while still in i32 (a sum of eight
9983 // i8 products still cannot overflow i32), and only then extend to i64 and add
9984 // to the accumulator.
9985 if (VT.getVectorElementType() == MVT::i64) {
9986 assert(Accum.getSimpleValueType() == VT);
9987 // vdot4a* reduces each group of four i8 lanes into one i32 lane, so the
9988 // intermediate i32 result has 1/4 the element count of the i8 inputs.
9989 MVT DotVT = MVT::getVectorVT(
9990 VT: MVT::i32, EC: ArgVT.getVectorElementCount().divideCoefficientBy(RHS: 4));
9991 SDValue Dot = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: DotVT,
9992 Ops: {DAG.getConstant(Val: 0, DL, VT: DotVT), A, B});
9993 // The reduced i32 sums are signed for SMLA/SUMLA and unsigned for UMLA.
9994 unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA
9995 ? ISD::ZERO_EXTEND
9996 : ISD::SIGN_EXTEND;
9997
9998 MVT NarrowVT = VT.changeVectorElementType(EltVT: MVT::i32);
9999 if (VT.isScalableVector() &&
10000 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT: NarrowVT))
10001 .second) {
10002 // When the accumulator is a single vector (LMUL 1), the i32 subvectors
10003 // are a fractional LMUL, so extracting the high subvector would need a
10004 // vslidedown. Widen the i32 sums to i64 first instead; the i64
10005 // subvectors are then register-aligned and the reduction is a plain add.
10006 MVT WideVT = DotVT.changeVectorElementType(EltVT: MVT::i64);
10007 SDValue Wide = DAG.getNode(Opcode: ExtOpc, DL, VT: WideVT, Operand: Dot);
10008 unsigned Stride = VT.getVectorMinNumElements();
10009 SDValue Sum = DAG.getExtractSubvector(DL, VT, Vec: Wide, Idx: 0);
10010 for (unsigned I = 1, E = WideVT.getVectorMinNumElements() / Stride;
10011 I != E; ++I)
10012 Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Sum,
10013 N2: DAG.getExtractSubvector(DL, VT, Vec: Wide, Idx: I * Stride));
10014 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Accum, N2: Sum);
10015 }
10016
10017 // Add the i32 partial sums down to the accumulator's element count by
10018 // extracting and summing the subvectors (still in i32 to avoid a wider
10019 // extend), then extend once to i64 and accumulate.
10020 unsigned Stride = NarrowVT.getVectorMinNumElements();
10021 SDValue Sum = DAG.getExtractSubvector(DL, VT: NarrowVT, Vec: Dot, Idx: 0);
10022 for (unsigned I = 1, E = DotVT.getVectorMinNumElements() / Stride; I != E;
10023 ++I)
10024 Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: NarrowVT, N1: Sum,
10025 N2: DAG.getExtractSubvector(DL, VT: NarrowVT, Vec: Dot, Idx: I * Stride));
10026 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Accum,
10027 N2: DAG.getNode(Opcode: ExtOpc, DL, VT, Operand: Sum));
10028 }
10029
10030 assert(Accum.getSimpleValueType() == VT &&
10031 VT.getVectorElementType() == MVT::i32);
10032
10033 // The zvdot4a8i pseudos are defined with sources and destination both
10034 // being i32. This cast is needed for correctness to avoid incorrect
10035 // .vx matching of i8 splats.
10036 A = DAG.getBitcast(VT, V: A);
10037 B = DAG.getBitcast(VT, V: B);
10038
10039 MVT ContainerVT = VT;
10040 if (VT.isFixedLengthVector()) {
10041 ContainerVT = getContainerForFixedLengthVector(VT);
10042 Accum = convertToScalableVector(VT: ContainerVT, V: Accum, DAG, Subtarget);
10043 A = convertToScalableVector(VT: ContainerVT, V: A, DAG, Subtarget);
10044 B = convertToScalableVector(VT: ContainerVT, V: B, DAG, Subtarget);
10045 }
10046
10047 unsigned Opc;
10048 switch (Op.getOpcode()) {
10049 case ISD::PARTIAL_REDUCE_SMLA:
10050 Opc = RISCVISD::VDOT4A_VL;
10051 break;
10052 case ISD::PARTIAL_REDUCE_UMLA:
10053 Opc = RISCVISD::VDOT4AU_VL;
10054 break;
10055 case ISD::PARTIAL_REDUCE_SUMLA:
10056 Opc = RISCVISD::VDOT4ASU_VL;
10057 break;
10058 default:
10059 llvm_unreachable("Unexpected opcode");
10060 }
10061 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
10062 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, Ops: {A, B, Accum, Mask, VL});
10063 if (VT.isFixedLengthVector())
10064 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
10065 return Res;
10066}
10067
10068static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty,
10069 SelectionDAG &DAG, unsigned Flags) {
10070 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
10071}
10072
10073static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
10074 SelectionDAG &DAG, unsigned Flags) {
10075 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
10076 TargetFlags: Flags);
10077}
10078
10079static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
10080 SelectionDAG &DAG, unsigned Flags) {
10081 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
10082 Offset: N->getOffset(), TargetFlags: Flags);
10083}
10084
10085static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
10086 SelectionDAG &DAG, unsigned Flags) {
10087 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
10088}
10089
10090static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL,
10091 EVT Ty, SelectionDAG &DAG) {
10092 RISCVConstantPoolValue *CPV = RISCVConstantPoolValue::Create(GV: N->getGlobal());
10093 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: Ty, Align: Align(8));
10094 SDValue LC = DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: CPAddr);
10095 return DAG.getLoad(
10096 VT: Ty, dl: DL, Chain: DAG.getEntryNode(), Ptr: LC,
10097 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
10098}
10099
10100static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL,
10101 EVT Ty, SelectionDAG &DAG) {
10102 RISCVConstantPoolValue *CPV =
10103 RISCVConstantPoolValue::Create(C&: *DAG.getContext(), S: N->getSymbol());
10104 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: Ty, Align: Align(8));
10105 SDValue LC = DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: CPAddr);
10106 return DAG.getLoad(
10107 VT: Ty, dl: DL, Chain: DAG.getEntryNode(), Ptr: LC,
10108 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
10109}
10110
10111template <class NodeTy>
10112SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
10113 bool IsLocal, bool IsExternWeak) const {
10114 SDLoc DL(N);
10115 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
10116
10117 // When HWASAN is used and tagging of global variables is enabled
10118 // they should be accessed via the GOT, since the tagged address of a global
10119 // is incompatible with existing code models. This also applies to non-pic
10120 // mode.
10121 if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) {
10122 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
10123 if (IsLocal && !Subtarget.allowTaggedGlobals())
10124 // Use PC-relative addressing to access the symbol. This generates the
10125 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
10126 // %pcrel_lo(auipc)).
10127 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
10128
10129 // Use PC-relative addressing to access the GOT for this symbol, then load
10130 // the address from the GOT. This generates the pattern (PseudoLGA sym),
10131 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
10132 SDValue Load =
10133 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLGA, dl: DL, VT: Ty, Op1: Addr), 0);
10134 MachineFunction &MF = DAG.getMachineFunction();
10135 MachineMemOperand *MemOp = MF.getMachineMemOperand(
10136 PtrInfo: MachinePointerInfo::getGOT(MF),
10137 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
10138 MachineMemOperand::MOInvariant,
10139 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
10140 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
10141 return Load;
10142 }
10143
10144 switch (getTargetMachine().getCodeModel()) {
10145 default:
10146 reportFatalUsageError(reason: "Unsupported code model for lowering");
10147 case CodeModel::Small: {
10148 // Generate a sequence for accessing addresses within the first 2 GiB of
10149 // address space.
10150 if (Subtarget.hasVendorXqcili()) {
10151 // Use QC.E.LI to generate the address, as this is easier to relax than
10152 // LUI/ADDI.
10153 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
10154 return DAG.getNode(Opcode: RISCVISD::QC_E_LI, DL, VT: Ty, Operand: Addr);
10155 }
10156
10157 // This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
10158 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
10159 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
10160 SDValue MNHi = DAG.getNode(Opcode: RISCVISD::HI, DL, VT: Ty, Operand: AddrHi);
10161 return DAG.getNode(Opcode: RISCVISD::ADD_LO, DL, VT: Ty, N1: MNHi, N2: AddrLo);
10162 }
10163 case CodeModel::Medium: {
10164 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
10165 if (IsExternWeak) {
10166 // An extern weak symbol may be undefined, i.e. have value 0, which may
10167 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
10168 // symbol. This generates the pattern (PseudoLGA sym), which expands to
10169 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
10170 SDValue Load =
10171 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLGA, dl: DL, VT: Ty, Op1: Addr), 0);
10172 MachineFunction &MF = DAG.getMachineFunction();
10173 MachineMemOperand *MemOp = MF.getMachineMemOperand(
10174 PtrInfo: MachinePointerInfo::getGOT(MF),
10175 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
10176 MachineMemOperand::MOInvariant,
10177 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
10178 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
10179 return Load;
10180 }
10181
10182 // Generate a sequence for accessing addresses within any 2GiB range within
10183 // the address space. This generates the pattern (PseudoLLA sym), which
10184 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
10185 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
10186 }
10187 case CodeModel::Large: {
10188 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N))
10189 return getLargeGlobalAddress(N: G, DL, Ty, DAG);
10190
10191 // Using pc-relative mode for other node type.
10192 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
10193 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
10194 }
10195 }
10196}
10197
10198SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
10199 SelectionDAG &DAG) const {
10200 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
10201 assert(N->getOffset() == 0 && "unexpected offset in global node");
10202 const GlobalValue *GV = N->getGlobal();
10203 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(GV);
10204 return getAddr(N, DAG, IsLocal, IsExternWeak: GV->hasExternalWeakLinkage());
10205}
10206
10207SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
10208 SelectionDAG &DAG) const {
10209 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Val&: Op);
10210
10211 return getAddr(N, DAG);
10212}
10213
10214SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
10215 SelectionDAG &DAG) const {
10216 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Val&: Op);
10217
10218 return getAddr(N, DAG);
10219}
10220
10221SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
10222 SelectionDAG &DAG) const {
10223 JumpTableSDNode *N = cast<JumpTableSDNode>(Val&: Op);
10224
10225 return getAddr(N, DAG);
10226}
10227
10228SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
10229 SelectionDAG &DAG,
10230 bool UseGOT) const {
10231 SDLoc DL(N);
10232 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
10233 const GlobalValue *GV = N->getGlobal();
10234 MVT XLenVT = Subtarget.getXLenVT();
10235
10236 if (UseGOT) {
10237 // Use PC-relative addressing to access the GOT for this TLS symbol, then
10238 // load the address from the GOT and add the thread pointer. This generates
10239 // the pattern (PseudoLA_TLS_IE sym), which expands to
10240 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
10241 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
10242 SDValue Load =
10243 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLS_IE, dl: DL, VT: Ty, Op1: Addr), 0);
10244 MachineFunction &MF = DAG.getMachineFunction();
10245 MachineMemOperand *MemOp = MF.getMachineMemOperand(
10246 PtrInfo: MachinePointerInfo::getGOT(MF),
10247 F: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
10248 MachineMemOperand::MOInvariant,
10249 MemTy: LLT(Ty.getSimpleVT()), BaseAlignment: Align(Ty.getFixedSizeInBits() / 8));
10250 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
10251
10252 // Add the thread pointer.
10253 SDValue TPReg = DAG.getRegister(Reg: RISCV::X4, VT: XLenVT);
10254 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Load, N2: TPReg);
10255 }
10256
10257 // Generate a sequence for accessing the address relative to the thread
10258 // pointer, with the appropriate adjustment for the thread pointer offset.
10259 // This generates the pattern
10260 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
10261 SDValue AddrHi =
10262 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_HI);
10263 SDValue AddrAdd =
10264 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_ADD);
10265 SDValue AddrLo =
10266 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_LO);
10267
10268 SDValue MNHi = DAG.getNode(Opcode: RISCVISD::HI, DL, VT: Ty, Operand: AddrHi);
10269 SDValue TPReg = DAG.getRegister(Reg: RISCV::X4, VT: XLenVT);
10270 SDValue MNAdd =
10271 DAG.getNode(Opcode: RISCVISD::ADD_TPREL, DL, VT: Ty, N1: MNHi, N2: TPReg, N3: AddrAdd);
10272 return DAG.getNode(Opcode: RISCVISD::ADD_LO, DL, VT: Ty, N1: MNAdd, N2: AddrLo);
10273}
10274
10275SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
10276 SelectionDAG &DAG) const {
10277 SDLoc DL(N);
10278 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
10279 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
10280 const GlobalValue *GV = N->getGlobal();
10281
10282 // Use a PC-relative addressing mode to access the global dynamic GOT address.
10283 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
10284 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
10285 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
10286 SDValue Load =
10287 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLS_GD, dl: DL, VT: Ty, Op1: Addr), 0);
10288
10289 // Prepare argument list to generate call.
10290 ArgListTy Args;
10291 Args.emplace_back(args&: Load, args&: CallTy);
10292
10293 // Setup call to __tls_get_addr.
10294 TargetLowering::CallLoweringInfo CLI(DAG);
10295 CLI.setDebugLoc(DL)
10296 .setChain(DAG.getEntryNode())
10297 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
10298 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
10299 ArgsList: std::move(Args));
10300
10301 return LowerCallTo(CLI).first;
10302}
10303
10304SDValue RISCVTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
10305 SelectionDAG &DAG) const {
10306 SDLoc DL(N);
10307 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
10308 const GlobalValue *GV = N->getGlobal();
10309
10310 // Use a PC-relative addressing mode to access the global dynamic GOT address.
10311 // This generates the pattern (PseudoLA_TLSDESC sym), which expands to
10312 //
10313 // auipc tX, %tlsdesc_hi(symbol) // R_RISCV_TLSDESC_HI20(symbol)
10314 // lw tY, tX, %tlsdesc_load_lo(label) // R_RISCV_TLSDESC_LOAD_LO12(label)
10315 // addi a0, tX, %tlsdesc_add_lo(label) // R_RISCV_TLSDESC_ADD_LO12(label)
10316 // jalr t0, tY // R_RISCV_TLSDESC_CALL(label)
10317 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
10318 return SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLSDESC, dl: DL, VT: Ty, Op1: Addr), 0);
10319}
10320
10321SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
10322 SelectionDAG &DAG) const {
10323 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
10324 assert(N->getOffset() == 0 && "unexpected offset in global node");
10325
10326 if (DAG.getTarget().useEmulatedTLS())
10327 return LowerToTLSEmulatedModel(GA: N, DAG);
10328
10329 TLSModel::Model Model = getTargetMachine().getTLSModel(GV: N->getGlobal());
10330
10331 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
10332 CallingConv::GHC)
10333 reportFatalUsageError(reason: "In GHC calling convention TLS is not supported");
10334
10335 SDValue Addr;
10336 switch (Model) {
10337 case TLSModel::LocalExec:
10338 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
10339 break;
10340 case TLSModel::InitialExec:
10341 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
10342 break;
10343 case TLSModel::LocalDynamic:
10344 case TLSModel::GeneralDynamic:
10345 Addr = DAG.getTarget().useTLSDESC() ? getTLSDescAddr(N, DAG)
10346 : getDynamicTLSAddr(N, DAG);
10347 break;
10348 }
10349
10350 return Addr;
10351}
10352
10353// Return true if Val is equal to (setcc LHS, RHS, CC).
10354// Return false if Val is the inverse of (setcc LHS, RHS, CC).
10355// Otherwise, return std::nullopt.
10356static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
10357 ISD::CondCode CC, SDValue Val) {
10358 assert(Val->getOpcode() == ISD::SETCC);
10359 SDValue LHS2 = Val.getOperand(i: 0);
10360 SDValue RHS2 = Val.getOperand(i: 1);
10361 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
10362
10363 if (LHS == LHS2 && RHS == RHS2) {
10364 if (CC == CC2)
10365 return true;
10366 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
10367 return false;
10368 } else if (LHS == RHS2 && RHS == LHS2) {
10369 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
10370 if (CC == CC2)
10371 return true;
10372 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
10373 return false;
10374 }
10375
10376 return std::nullopt;
10377}
10378
10379static bool isSimm12Constant(SDValue V) {
10380 return isa<ConstantSDNode>(Val: V) && V->getAsAPIntVal().isSignedIntN(N: 12);
10381}
10382
10383static SDValue lowerSelectToBinOp(SDNode *N, SelectionDAG &DAG,
10384 const RISCVSubtarget &Subtarget) {
10385 SDValue CondV = N->getOperand(Num: 0);
10386 SDValue TrueV = N->getOperand(Num: 1);
10387 SDValue FalseV = N->getOperand(Num: 2);
10388 MVT VT = N->getSimpleValueType(ResNo: 0);
10389 SDLoc DL(N);
10390
10391 if (!Subtarget.hasConditionalMoveFusion()) {
10392 // (select c, -1, y) -> -c | y
10393 if (isAllOnesConstant(V: TrueV)) {
10394 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
10395 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
10396 }
10397 // (select c, y, -1) -> (c-1) | y
10398 if (isAllOnesConstant(V: FalseV)) {
10399 SDValue Neg = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV,
10400 N2: DAG.getAllOnesConstant(DL, VT));
10401 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
10402 }
10403
10404 const bool HasZicond = VT.isScalarInteger() && Subtarget.hasStdExtZicond();
10405
10406 // (select c, 0, y) -> (c-1) & y
10407 if (isNullConstant(V: TrueV) && (!HasZicond || isSimm12Constant(V: FalseV))) {
10408 SDValue Neg =
10409 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
10410 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
10411 }
10412 if (isNullConstant(V: FalseV)) {
10413 if (auto *TrueC = dyn_cast<ConstantSDNode>(Val&: TrueV)) {
10414 // (select c, y, 0) -> (c * (y - 1)) + c
10415 int64_t MulImm = TrueC->getSExtValue();
10416 if (MulImm != INT64_MIN && isInt<12>(x: MulImm - 1) &&
10417 Subtarget.hasVendorXqciac())
10418 return DAG.getNode(Opcode: RISCVISD::QC_MULIADD, DL, VT, N1: CondV, N2: CondV,
10419 N3: DAG.getSignedTargetConstant(Val: MulImm - 1, DL, VT));
10420
10421 // (select c, (1 << ShAmount) + 1, 0) -> (c << ShAmount) + c
10422 uint64_t TrueM1 = TrueC->getZExtValue() - 1;
10423 if (isPowerOf2_64(Value: TrueM1)) {
10424 unsigned ShAmount = Log2_64(Value: TrueM1);
10425 if (Subtarget.hasShlAdd(ShAmt: ShAmount))
10426 return DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: CondV,
10427 N2: DAG.getTargetConstant(Val: ShAmount, DL, VT), N3: CondV);
10428 }
10429 }
10430 // (select c, y, 0) -> -c & y
10431 if (!HasZicond || isSimm12Constant(V: TrueV)) {
10432 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
10433 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
10434 }
10435 }
10436 }
10437
10438 // select c, ~x, x --> xor -c, x
10439 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
10440 const APInt &TrueVal = TrueV->getAsAPIntVal();
10441 const APInt &FalseVal = FalseV->getAsAPIntVal();
10442 if (~TrueVal == FalseVal) {
10443 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
10444 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
10445 }
10446 }
10447
10448 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
10449 // when both truev and falsev are also setcc.
10450 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
10451 FalseV.getOpcode() == ISD::SETCC) {
10452 SDValue LHS = CondV.getOperand(i: 0);
10453 SDValue RHS = CondV.getOperand(i: 1);
10454 ISD::CondCode CC = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10455
10456 // (select x, x, y) -> x | y
10457 // (select !x, x, y) -> x & y
10458 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
10459 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
10460 N2: DAG.getFreeze(V: FalseV));
10461 }
10462 // (select x, y, x) -> x & y
10463 // (select !x, y, x) -> x | y
10464 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
10465 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
10466 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
10467 }
10468 }
10469
10470 return SDValue();
10471}
10472
10473// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
10474// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
10475// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
10476// being `0` or `-1`. In such cases we can replace `select` with `and`.
10477// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
10478// than `c0`?
10479static SDValue
10480foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
10481 const RISCVSubtarget &Subtarget) {
10482 if (Subtarget.hasShortForwardBranchIALU())
10483 return SDValue();
10484
10485 unsigned SelOpNo = 0;
10486 SDValue Sel = BO->getOperand(Num: 0);
10487 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
10488 SelOpNo = 1;
10489 Sel = BO->getOperand(Num: 1);
10490 }
10491
10492 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
10493 return SDValue();
10494
10495 unsigned ConstSelOpNo = 1;
10496 unsigned OtherSelOpNo = 2;
10497 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
10498 ConstSelOpNo = 2;
10499 OtherSelOpNo = 1;
10500 }
10501 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
10502 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
10503 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
10504 return SDValue();
10505
10506 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
10507 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
10508 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
10509 return SDValue();
10510
10511 SDLoc DL(Sel);
10512 EVT VT = BO->getValueType(ResNo: 0);
10513
10514 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
10515 if (SelOpNo == 1)
10516 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
10517
10518 SDValue NewConstOp =
10519 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
10520 if (!NewConstOp)
10521 return SDValue();
10522
10523 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
10524 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
10525 return SDValue();
10526
10527 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
10528 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
10529 if (SelOpNo == 1)
10530 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
10531 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
10532
10533 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
10534 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
10535 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
10536}
10537
10538SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
10539 SDValue CondV = Op.getOperand(i: 0);
10540 SDValue TrueV = Op.getOperand(i: 1);
10541 SDValue FalseV = Op.getOperand(i: 2);
10542 SDLoc DL(Op);
10543 MVT VT = Op.getSimpleValueType();
10544 MVT XLenVT = Subtarget.getXLenVT();
10545
10546 // Handle P extension packed types by bitcasting to an integer of
10547 // matching width and reusing the scalar selection mechanism.
10548 // Reachable cases:
10549 // RV32: v4i8/v2i16 -> select on i32
10550 // RV32: v8i8/v4i16 -> select on i64 (legalizes to two i32 selects)
10551 // RV64: v8i8/v4i16/v2i32 -> select on i64
10552 if (Subtarget.isPExtPackedType(VT)) {
10553 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
10554 SDValue TrueVInt = DAG.getBitcast(VT: IntVT, V: TrueV);
10555 SDValue FalseVInt = DAG.getBitcast(VT: IntVT, V: FalseV);
10556 SDValue ResultInt =
10557 DAG.getNode(Opcode: ISD::SELECT, DL, VT: IntVT, N1: CondV, N2: TrueVInt, N3: FalseVInt);
10558 return DAG.getBitcast(VT, V: ResultInt);
10559 }
10560
10561 // Lower vector SELECTs to VSELECTs by splatting the condition.
10562 if (VT.isVector()) {
10563 MVT SplatCondVT = VT.changeVectorElementType(EltVT: MVT::i1);
10564 SDValue CondSplat = DAG.getSplat(VT: SplatCondVT, DL, Op: CondV);
10565 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CondSplat, N2: TrueV, N3: FalseV);
10566 }
10567
10568 // Try some other optimizations before falling back to generic lowering.
10569 if (SDValue V = lowerSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
10570 return V;
10571
10572 // When there is no cost for GPR <-> FPR, we can use zicond select for
10573 // floating value when CondV is int type
10574 bool FPinGPR = Subtarget.hasStdExtZfinx();
10575
10576 // We can handle FGPR without spliting into hi/lo parts
10577 bool FitsInGPR = TypeSize::isKnownLE(LHS: VT.getSizeInBits(),
10578 RHS: Subtarget.getXLenVT().getSizeInBits());
10579
10580 bool UseZicondForFPSel = Subtarget.hasStdExtZicond() && FPinGPR &&
10581 VT.isFloatingPoint() && FitsInGPR;
10582
10583 if (UseZicondForFPSel) {
10584
10585 auto CastToInt = [&](SDValue V) -> SDValue {
10586 // Treat +0.0 as int 0 to enable single 'czero' instruction generation.
10587 if (isNullFPConstant(V))
10588 return DAG.getConstant(Val: 0, DL, VT: XLenVT);
10589
10590 if (VT == MVT::f16)
10591 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: V);
10592
10593 if (VT == MVT::f32 && Subtarget.is64Bit())
10594 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: XLenVT, Operand: V);
10595
10596 return DAG.getBitcast(VT: XLenVT, V);
10597 };
10598
10599 SDValue TrueVInt = CastToInt(TrueV);
10600 SDValue FalseVInt = CastToInt(FalseV);
10601
10602 // Emit integer SELECT (lowers to Zicond)
10603 SDValue ResultInt =
10604 DAG.getNode(Opcode: ISD::SELECT, DL, VT: XLenVT, N1: CondV, N2: TrueVInt, N3: FalseVInt);
10605
10606 // Convert back to floating VT
10607 if (VT == MVT::f32 && Subtarget.is64Bit())
10608 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT, Operand: ResultInt);
10609
10610 if (VT == MVT::f16)
10611 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: ResultInt);
10612
10613 return DAG.getBitcast(VT, V: ResultInt);
10614 }
10615
10616 // When Zicond is present, emit CZERO_EQZ and CZERO_NEZ
10617 // nodes to implement the SELECT. Performing the lowering here allows for
10618 // greater control over when CZERO_{EQZ/NEZ} are used vs another branchless
10619 // sequence or RISCVISD::SELECT_CC node (branch-based select).
10620 if (Subtarget.hasStdExtZicond() && VT.isScalarInteger()) {
10621
10622 // (select c, t, 0) -> (czero_eqz t, c)
10623 if (isNullConstant(V: FalseV))
10624 return DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV);
10625 // (select c, 0, f) -> (czero_nez f, c)
10626 if (isNullConstant(V: TrueV))
10627 return DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV);
10628
10629 // Check to see if a given operation is a 'NOT', if so return the negated
10630 // operand
10631 auto getNotOperand = [](const SDValue &Op) -> std::optional<const SDValue> {
10632 using namespace llvm::SDPatternMatch;
10633 SDValue Xor;
10634 if (sd_match(N: Op, P: m_OneUse(P: m_Not(V: m_Value(N&: Xor))))) {
10635 return Xor;
10636 }
10637 return std::nullopt;
10638 };
10639 // (select c, (and f, x), f) -> (or (and f, x), (czero_nez f, c))
10640 // (select c, (and f, ~x), f) -> (andn f, (czero_eqz x, c))
10641 if (TrueV.getOpcode() == ISD::AND &&
10642 (TrueV.getOperand(i: 0) == FalseV || TrueV.getOperand(i: 1) == FalseV)) {
10643 auto NotOperand = (TrueV.getOperand(i: 0) == FalseV)
10644 ? getNotOperand(TrueV.getOperand(i: 1))
10645 : getNotOperand(TrueV.getOperand(i: 0));
10646 if (NotOperand) {
10647 SDValue CMOV =
10648 DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: *NotOperand, N2: CondV);
10649 SDValue NOT = DAG.getNOT(DL, Val: CMOV, VT);
10650 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseV, N2: NOT);
10651 }
10652 return DAG.getNode(
10653 Opcode: ISD::OR, DL, VT, N1: TrueV,
10654 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV));
10655 }
10656
10657 // (select c, t, (and t, x)) -> (or (czero_eqz t, c), (and t, x))
10658 // (select c, t, (and t, ~x)) -> (andn t, (czero_nez x, c))
10659 if (FalseV.getOpcode() == ISD::AND &&
10660 (FalseV.getOperand(i: 0) == TrueV || FalseV.getOperand(i: 1) == TrueV)) {
10661 auto NotOperand = (FalseV.getOperand(i: 0) == TrueV)
10662 ? getNotOperand(FalseV.getOperand(i: 1))
10663 : getNotOperand(FalseV.getOperand(i: 0));
10664 if (NotOperand) {
10665 SDValue CMOV =
10666 DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: *NotOperand, N2: CondV);
10667 SDValue NOT = DAG.getNOT(DL, Val: CMOV, VT);
10668 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueV, N2: NOT);
10669 }
10670 return DAG.getNode(
10671 Opcode: ISD::OR, DL, VT, N1: FalseV,
10672 N2: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV));
10673 }
10674
10675 // (select c, c1, c2) -> (add (czero_nez c2 - c1, c), c1)
10676 // (select c, c1, c2) -> (add (czero_eqz c1 - c2, c), c2)
10677 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
10678 const APInt &TrueVal = TrueV->getAsAPIntVal();
10679 const APInt &FalseVal = FalseV->getAsAPIntVal();
10680
10681 // Prefer these over Zicond to avoid materializing an immediate:
10682 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
10683 // (select (x > -1), z, y) -> x >> (XLEN - 1) & (y - z) + z
10684 if (CondV.getOpcode() == ISD::SETCC &&
10685 CondV.getOperand(i: 0).getValueType() == VT && CondV.hasOneUse()) {
10686 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10687 if ((CCVal == ISD::SETLT && isNullConstant(V: CondV.getOperand(i: 1))) ||
10688 (CCVal == ISD::SETGT && isAllOnesConstant(V: CondV.getOperand(i: 1)))) {
10689 int64_t TrueImm = TrueVal.getSExtValue();
10690 int64_t FalseImm = FalseVal.getSExtValue();
10691 if (CCVal == ISD::SETGT)
10692 std::swap(a&: TrueImm, b&: FalseImm);
10693 if (isInt<12>(x: TrueImm) && isInt<12>(x: FalseImm) &&
10694 isInt<12>(x: TrueImm - FalseImm)) {
10695 SDValue SRA =
10696 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CondV.getOperand(i: 0),
10697 N2: DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT));
10698 SDValue AND =
10699 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
10700 N2: DAG.getSignedConstant(Val: TrueImm - FalseImm, DL, VT));
10701 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND,
10702 N2: DAG.getSignedConstant(Val: FalseImm, DL, VT));
10703 }
10704 }
10705 }
10706
10707 // Use SLLI/ADDI (and possible XORI) to avoid having to materialize
10708 // a constant in register
10709 if ((TrueVal - FalseVal).isPowerOf2() && FalseVal.isSignedIntN(N: 12)) {
10710 SDValue Log2 = DAG.getConstant(Val: (TrueVal - FalseVal).logBase2(), DL, VT);
10711 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10712 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BitDiff, N2: FalseV);
10713 }
10714 if ((FalseVal - TrueVal).isPowerOf2() && TrueVal.isSignedIntN(N: 12)) {
10715 SDValue Log2 = DAG.getConstant(Val: (FalseVal - TrueVal).logBase2(), DL, VT);
10716 CondV = DAG.getLogicalNOT(DL, Val: CondV, VT: CondV.getValueType());
10717 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10718 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BitDiff, N2: TrueV);
10719 }
10720
10721 // If we can't use ADDI, it might still be profitable to use SLLI/ADD or
10722 // SLLI/SUB. We need to materialize a large constant for the czero+add
10723 // sequence below anyway. Using SLLI avoids materializing the delta.
10724 // Don't do this if the condition is an equality comparison since czero
10725 // allows us to fold part of the compare.
10726 if ((TrueVal - FalseVal).isPowerOf2() &&
10727 !(CondV.getOpcode() == ISD::SETCC &&
10728 ISD::isIntEqualitySetCC(
10729 Code: cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get()))) {
10730 SDValue Log2 = DAG.getConstant(Val: (TrueVal - FalseVal).logBase2(), DL, VT);
10731 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10732 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FalseV, N2: BitDiff);
10733 }
10734 if ((FalseVal - TrueVal).isPowerOf2() && !FalseVal.isSignedIntN(N: 12) &&
10735 !(CondV.getOpcode() == ISD::SETCC &&
10736 ISD::isIntEqualitySetCC(
10737 Code: cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get()))) {
10738 SDValue Log2 = DAG.getConstant(Val: (FalseVal - TrueVal).logBase2(), DL, VT);
10739 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10740 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: BitDiff);
10741 }
10742
10743 auto getCost = [&](const APInt &Delta, const APInt &Addend) {
10744 const int DeltaCost = RISCVMatInt::getIntMatCost(
10745 Val: Delta, Size: Subtarget.getXLen(), STI: Subtarget, /*CompressionCost=*/true);
10746 // Does the addend fold into an ADDI
10747 if (Addend.isSignedIntN(N: 12))
10748 return DeltaCost;
10749 const int AddendCost = RISCVMatInt::getIntMatCost(
10750 Val: Addend, Size: Subtarget.getXLen(), STI: Subtarget, /*CompressionCost=*/true);
10751 return AddendCost + DeltaCost;
10752 };
10753 bool IsCZERO_NEZ = getCost(FalseVal - TrueVal, TrueVal) <=
10754 getCost(TrueVal - FalseVal, FalseVal);
10755 SDValue LHSVal = DAG.getConstant(
10756 Val: IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
10757 SDValue CMOV =
10758 DAG.getNode(Opcode: IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10759 DL, VT, N1: LHSVal, N2: CondV);
10760 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CMOV, N2: IsCZERO_NEZ ? TrueV : FalseV);
10761 }
10762
10763 // (select c, c1, t) -> (add (czero_nez t - c1, c), c1)
10764 // (select c, t, c1) -> (add (czero_eqz t - c1, c), c1)
10765 if (isa<ConstantSDNode>(Val: TrueV) != isa<ConstantSDNode>(Val: FalseV)) {
10766 bool IsCZERO_NEZ = isa<ConstantSDNode>(Val: TrueV);
10767 SDValue ConstVal = IsCZERO_NEZ ? TrueV : FalseV;
10768 SDValue RegV = IsCZERO_NEZ ? FalseV : TrueV;
10769 int64_t RawConstVal = cast<ConstantSDNode>(Val&: ConstVal)->getSExtValue();
10770 // Efficient only if the constant and its negation fit into `ADDI`
10771 // Prefer Add/Sub over Xor since can be compressed for small immediates
10772 if (isInt<12>(x: RawConstVal)) {
10773 // Fall back to XORI if Const == -0x800 since we don't have SUBI.
10774 unsigned SubOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::SUB;
10775 unsigned AddOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::ADD;
10776 SDValue SubOp = DAG.getNode(Opcode: SubOpc, DL, VT, N1: RegV, N2: ConstVal);
10777 SDValue CZERO =
10778 DAG.getNode(Opcode: IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10779 DL, VT, N1: SubOp, N2: CondV);
10780 return DAG.getNode(Opcode: AddOpc, DL, VT, N1: CZERO, N2: ConstVal);
10781 }
10782 }
10783
10784 // Unless we have the short forward branch optimization, or we are
10785 // optimizing for size.
10786 if (!Subtarget.hasConditionalMoveFusion() && !DAG.shouldOptForSize()) {
10787 using namespace llvm::SDPatternMatch;
10788
10789 // (select c, x, -x) -> (sub (czero_eqz x, c), (czero_nez x, c))
10790 if (sd_match(N: FalseV, P: m_Neg(V: m_Specific(N: TrueV)))) {
10791 TrueV = DAG.getFreeze(V: TrueV);
10792 return DAG.getNode(
10793 Opcode: ISD::SUB, DL, VT,
10794 N1: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV),
10795 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: TrueV, N2: CondV));
10796 }
10797 // (select c, -x, x) -> (sub (czero_nez x, c), (czero_eqz x, c))
10798 if (sd_match(N: TrueV, P: m_Neg(V: m_Specific(N: FalseV)))) {
10799 FalseV = DAG.getFreeze(V: FalseV);
10800 return DAG.getNode(
10801 Opcode: ISD::SUB, DL, VT,
10802 N1: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV),
10803 N2: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: FalseV, N2: CondV));
10804 }
10805
10806 // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
10807 return DAG.getNode(
10808 Opcode: ISD::OR, DL, VT,
10809 N1: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV),
10810 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV),
10811 Flags: SDNodeFlags::Disjoint);
10812 }
10813 }
10814
10815 if (Op.hasOneUse()) {
10816 unsigned UseOpc = Op->user_begin()->getOpcode();
10817 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
10818 SDNode *BinOp = *Op->user_begin();
10819 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
10820 DAG, Subtarget)) {
10821 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
10822 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
10823 // may return a constant node and cause crash in lowerSELECT.
10824 if (NewSel.getOpcode() == ISD::SELECT)
10825 return lowerSELECT(Op: NewSel, DAG);
10826 return NewSel;
10827 }
10828 }
10829 }
10830
10831 // (select cc, 1.0, 0.0) -> (sint_to_fp (zext cc))
10832 // (select cc, 0.0, 1.0) -> (sint_to_fp (zext (xor cc, 1)))
10833 const ConstantFPSDNode *FPTV = dyn_cast<ConstantFPSDNode>(Val&: TrueV);
10834 const ConstantFPSDNode *FPFV = dyn_cast<ConstantFPSDNode>(Val&: FalseV);
10835 if (FPTV && FPFV) {
10836 if (FPTV->isOne() && FPFV->isPosZero())
10837 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: CondV);
10838 if (FPTV->isPosZero() && FPFV->isOne()) {
10839 SDValue XOR = DAG.getNode(Opcode: ISD::XOR, DL, VT: XLenVT, N1: CondV,
10840 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
10841 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: XOR);
10842 }
10843 }
10844
10845 // If the condition is not an integer SETCC which operates on XLenVT, we need
10846 // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
10847 // (select condv, truev, falsev)
10848 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
10849 if (CondV.getOpcode() != ISD::SETCC ||
10850 CondV.getOperand(i: 0).getSimpleValueType() != XLenVT) {
10851 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
10852 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
10853
10854 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
10855
10856 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, Ops);
10857 }
10858
10859 // If the CondV is the output of a SETCC node which operates on XLenVT inputs,
10860 // then merge the SETCC node into the lowered RISCVISD::SELECT_CC to take
10861 // advantage of the integer compare+branch instructions. i.e.:
10862 // (select (setcc lhs, rhs, cc), truev, falsev)
10863 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
10864 SDValue LHS = CondV.getOperand(i: 0);
10865 SDValue RHS = CondV.getOperand(i: 1);
10866 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10867
10868 // Special case for a select of 2 constants that have a difference of 1.
10869 // Normally this is done by DAGCombine, but if the select is introduced by
10870 // type legalization or op legalization, we miss it. Restricting to SETLT
10871 // case for now because that is what signed saturating add/sub need.
10872 // FIXME: We don't need the condition to be SETLT or even a SETCC,
10873 // but we would probably want to swap the true/false values if the condition
10874 // is SETGE/SETLE to avoid an XORI.
10875 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
10876 CCVal == ISD::SETLT) {
10877 const APInt &TrueVal = TrueV->getAsAPIntVal();
10878 const APInt &FalseVal = FalseV->getAsAPIntVal();
10879 if (TrueVal - 1 == FalseVal)
10880 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
10881 if (TrueVal + 1 == FalseVal)
10882 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
10883 }
10884
10885 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
10886 // 1 < x ? x : 1 -> 0 < x ? x : 1
10887 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
10888 RHS == TrueV && LHS == FalseV) {
10889 LHS = DAG.getConstant(Val: 0, DL, VT);
10890 // 0 <u x is the same as x != 0.
10891 if (CCVal == ISD::SETULT) {
10892 std::swap(a&: LHS, b&: RHS);
10893 CCVal = ISD::SETNE;
10894 }
10895 }
10896
10897 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
10898 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
10899 RHS == FalseV) {
10900 RHS = DAG.getConstant(Val: 0, DL, VT);
10901 }
10902
10903 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
10904
10905 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
10906 // (select (setcc lhs, rhs, CC), constant, falsev)
10907 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
10908 std::swap(a&: TrueV, b&: FalseV);
10909 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
10910 }
10911
10912 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
10913 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, Ops);
10914}
10915
10916SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
10917 SDValue CondV = Op.getOperand(i: 1);
10918 SDLoc DL(Op);
10919 MVT XLenVT = Subtarget.getXLenVT();
10920
10921 if (CondV.getOpcode() == ISD::SETCC &&
10922 CondV.getOperand(i: 0).getValueType() == XLenVT) {
10923 SDValue LHS = CondV.getOperand(i: 0);
10924 SDValue RHS = CondV.getOperand(i: 1);
10925 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10926
10927 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
10928
10929 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
10930 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
10931 N2: LHS, N3: RHS, N4: TargetCC, N5: Op.getOperand(i: 2));
10932 }
10933
10934 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
10935 N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: XLenVT),
10936 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
10937}
10938
10939SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
10940 MachineFunction &MF = DAG.getMachineFunction();
10941 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
10942
10943 SDLoc DL(Op);
10944 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
10945 VT: getPointerTy(DL: MF.getDataLayout()));
10946
10947 // vastart just stores the address of the VarArgsFrameIndex slot into the
10948 // memory location argument.
10949 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
10950 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
10951 PtrInfo: MachinePointerInfo(SV));
10952}
10953
10954SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
10955 SelectionDAG &DAG) const {
10956 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10957 MachineFunction &MF = DAG.getMachineFunction();
10958 MachineFrameInfo &MFI = MF.getFrameInfo();
10959 MFI.setFrameAddressIsTaken(true);
10960 Register FrameReg = RI.getFrameRegister(MF);
10961 int XLenInBytes = Subtarget.getXLen() / 8;
10962
10963 EVT VT = Op.getValueType();
10964 SDLoc DL(Op);
10965 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
10966 unsigned Depth = Op.getConstantOperandVal(i: 0);
10967 while (Depth--) {
10968 int Offset = -(XLenInBytes * 2);
10969 SDValue Ptr = DAG.getNode(
10970 Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
10971 N2: DAG.getSignedConstant(Val: Offset, DL, VT: getPointerTy(DL: DAG.getDataLayout())));
10972 FrameAddr =
10973 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
10974 }
10975 return FrameAddr;
10976}
10977
10978SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
10979 SelectionDAG &DAG) const {
10980 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10981 MachineFunction &MF = DAG.getMachineFunction();
10982 MachineFrameInfo &MFI = MF.getFrameInfo();
10983 MFI.setReturnAddressIsTaken(true);
10984 MVT XLenVT = Subtarget.getXLenVT();
10985 int XLenInBytes = Subtarget.getXLen() / 8;
10986
10987 EVT VT = Op.getValueType();
10988 SDLoc DL(Op);
10989 unsigned Depth = Op.getConstantOperandVal(i: 0);
10990 if (Depth) {
10991 int Off = -XLenInBytes;
10992 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
10993 SDValue Offset = DAG.getSignedConstant(Val: Off, DL, VT);
10994 return DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(),
10995 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr, N2: Offset),
10996 PtrInfo: MachinePointerInfo());
10997 }
10998
10999 // Return the value of the return address register, marking it an implicit
11000 // live-in.
11001 Register Reg = MF.addLiveIn(PReg: RI.getRARegister(), RC: getRegClassFor(VT: XLenVT));
11002 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT: XLenVT);
11003}
11004
11005SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
11006 SelectionDAG &DAG) const {
11007 SDLoc DL(Op);
11008 SDValue Lo = Op.getOperand(i: 0);
11009 SDValue Hi = Op.getOperand(i: 1);
11010 SDValue Shamt = Op.getOperand(i: 2);
11011 EVT VT = Lo.getValueType();
11012 unsigned XLen = Subtarget.getXLen();
11013
11014 // With P extension, use SLX (FSHL) for the high part.
11015 if (Subtarget.hasStdExtP()) {
11016 // HiRes = fshl(Hi, Lo, Shamt) - correct when Shamt < XLen
11017 SDValue HiRes = DAG.getNode(Opcode: ISD::FSHL, DL, VT, N1: Hi, N2: Lo, N3: Shamt);
11018 // LoRes = Lo << Shamt - correct Lo when Shamt < XLen,
11019 // Mask shift amount to avoid UB when Shamt >= XLen.
11020 SDValue ShamtMasked =
11021 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shamt, N2: DAG.getConstant(Val: XLen - 1, DL, VT));
11022 SDValue LoRes = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMasked);
11023
11024 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
11025 // FIXME: We should use a select and let LowerSelect make the
11026 // optimizations.
11027 SDValue ShAmtExt =
11028 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Shamt,
11029 N2: DAG.getConstant(Val: XLen - Log2_32(Value: XLen) - 1, DL, VT));
11030 SDValue Mask = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: ShAmtExt,
11031 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
11032
11033 // When Shamt >= XLen: HiRes = LoRes, LoRes = 0
11034 // HiRes = (HiRes & ~Mask) | (LoRes & Mask)
11035 SDValue HiMasked =
11036 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: DAG.getNOT(DL, Val: Mask, VT));
11037 SDValue LoMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: Mask);
11038 HiRes =
11039 DAG.getNode(Opcode: ISD::OR, DL, VT, N1: HiMasked, N2: LoMasked, Flags: SDNodeFlags::Disjoint);
11040
11041 // LoRes = LoRes & ~Mask (clear when Shamt >= XLen)
11042 LoRes = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: DAG.getNOT(DL, Val: Mask, VT));
11043
11044 return DAG.getMergeValues(Ops: {LoRes, HiRes}, dl: DL);
11045 }
11046
11047 // if Shamt-XLEN < 0: // Shamt < XLEN
11048 // Lo = Lo << Shamt
11049 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
11050 // else:
11051 // Lo = 0
11052 // Hi = Lo << (Shamt-XLEN)
11053
11054 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11055 SDValue One = DAG.getConstant(Val: 1, DL, VT);
11056 SDValue MinusXLen = DAG.getSignedConstant(Val: -(int)XLen, DL, VT);
11057 SDValue XLenMinus1 = DAG.getConstant(Val: XLen - 1, DL, VT);
11058 SDValue ShamtMinusXLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusXLen);
11059 SDValue XLenMinus1Shamt = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: XLenMinus1, N2: Shamt);
11060
11061 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
11062 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
11063 SDValue ShiftRightLo =
11064 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: XLenMinus1Shamt);
11065 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
11066 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
11067 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusXLen);
11068
11069 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusXLen, RHS: Zero, Cond: ISD::SETLT);
11070
11071 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
11072 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
11073
11074 SDValue Parts[2] = {Lo, Hi};
11075 return DAG.getMergeValues(Ops: Parts, dl: DL);
11076}
11077
11078SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
11079 bool IsSRA) const {
11080 SDLoc DL(Op);
11081 SDValue Lo = Op.getOperand(i: 0);
11082 SDValue Hi = Op.getOperand(i: 1);
11083 SDValue Shamt = Op.getOperand(i: 2);
11084 EVT VT = Lo.getValueType();
11085
11086 // With P extension, use NSRL/NSRA for RV32 or FSHR (SRX) for RV64.
11087 if (Subtarget.hasStdExtP()) {
11088 unsigned XLen = Subtarget.getXLen();
11089
11090 SDValue LoRes;
11091 if (Subtarget.is64Bit()) {
11092 // On RV64, use FSHR (SRX instruction) for the low part. We will need
11093 // to fix this later if ShAmt >= 64.
11094 LoRes = DAG.getNode(Opcode: ISD::FSHR, DL, VT, N1: Hi, N2: Lo, N3: Shamt);
11095 } else {
11096 // On RV32, use NSRL/NSRA for the low part.
11097 // NSRL/NSRA read 6 bits of shift amount, so they handle Shamt >= 32
11098 // correctly.
11099 LoRes = DAG.getNode(Opcode: IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL, VT, N1: Lo,
11100 N2: Hi, N3: Shamt);
11101 }
11102
11103 // Mask shift amount to avoid UB when Shamt >= XLen.
11104 SDValue ShamtMasked =
11105 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shamt, N2: DAG.getConstant(Val: XLen - 1, DL, VT));
11106 SDValue HiRes =
11107 DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL, VT, N1: Hi, N2: ShamtMasked);
11108
11109 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
11110 // FIXME: We should use a select and let LowerSelect make the
11111 // optimizations.
11112 SDValue ShAmtExt =
11113 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Shamt,
11114 N2: DAG.getConstant(Val: XLen - Log2_32(Value: XLen) - 1, DL, VT));
11115 SDValue Mask = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: ShAmtExt,
11116 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
11117
11118 if (Subtarget.is64Bit()) {
11119 // On RV64, FSHR masks shift amount to 63. We need to replace LoRes
11120 // with HiRes when Shamt >= 64.
11121 // LoRes = (LoRes & ~Mask) | (HiRes & Mask)
11122 SDValue LoMasked =
11123 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: DAG.getNOT(DL, Val: Mask, VT));
11124 SDValue HiMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: Mask);
11125 LoRes = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoMasked, N2: HiMasked,
11126 Flags: SDNodeFlags::Disjoint);
11127 }
11128
11129 // If ShAmt >= XLen, we need to replace HiRes with 0 or sign bits.
11130 if (IsSRA) {
11131 // sra hi, hi, (mask & (XLen-1)) - shifts by XLen-1 when shamt >= XLen
11132 SDValue MaskAmt = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mask,
11133 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
11134 HiRes = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: HiRes, N2: MaskAmt);
11135 } else {
11136 // andn hi, hi, mask - clears hi when shamt >= XLen
11137 HiRes = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: DAG.getNOT(DL, Val: Mask, VT));
11138 }
11139
11140 return DAG.getMergeValues(Ops: {LoRes, HiRes}, dl: DL);
11141 }
11142
11143 // SRA expansion:
11144 // if Shamt-XLEN < 0: // Shamt < XLEN
11145 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
11146 // Hi = Hi >>s Shamt
11147 // else:
11148 // Lo = Hi >>s (Shamt-XLEN);
11149 // Hi = Hi >>s (XLEN-1)
11150 //
11151 // SRL expansion:
11152 // if Shamt-XLEN < 0: // Shamt < XLEN
11153 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
11154 // Hi = Hi >>u Shamt
11155 // else:
11156 // Lo = Hi >>u (Shamt-XLEN);
11157 // Hi = 0;
11158
11159 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
11160
11161 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11162 SDValue One = DAG.getConstant(Val: 1, DL, VT);
11163 SDValue MinusXLen = DAG.getSignedConstant(Val: -(int)Subtarget.getXLen(), DL, VT);
11164 SDValue XLenMinus1 = DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT);
11165 SDValue ShamtMinusXLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusXLen);
11166 SDValue XLenMinus1Shamt = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: XLenMinus1, N2: Shamt);
11167
11168 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
11169 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
11170 SDValue ShiftLeftHi =
11171 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: XLenMinus1Shamt);
11172 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
11173 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
11174 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusXLen);
11175 SDValue HiFalse =
11176 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: XLenMinus1) : Zero;
11177
11178 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusXLen, RHS: Zero, Cond: ISD::SETLT);
11179
11180 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
11181 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
11182
11183 SDValue Parts[2] = {Lo, Hi};
11184 return DAG.getMergeValues(Ops: Parts, dl: DL);
11185}
11186
11187// Lower splats of i1 types to SETCC. For each mask vector type, we have a
11188// legal equivalently-sized i8 type, so we can use that as a go-between.
11189SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
11190 SelectionDAG &DAG) const {
11191 SDLoc DL(Op);
11192 MVT VT = Op.getSimpleValueType();
11193 SDValue SplatVal = Op.getOperand(i: 0);
11194 // All-zeros or all-ones splats are handled specially.
11195 if (ISD::isConstantSplatVectorAllOnes(N: Op.getNode())) {
11196 SDValue VL = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget).second;
11197 return DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT, Operand: VL);
11198 }
11199 if (ISD::isConstantSplatVectorAllZeros(N: Op.getNode())) {
11200 SDValue VL = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget).second;
11201 return DAG.getNode(Opcode: RISCVISD::VMCLR_VL, DL, VT, Operand: VL);
11202 }
11203 MVT InterVT = VT.changeVectorElementType(EltVT: MVT::i8);
11204 SplatVal = DAG.getNode(Opcode: ISD::AND, DL, VT: SplatVal.getValueType(), N1: SplatVal,
11205 N2: DAG.getConstant(Val: 1, DL, VT: SplatVal.getValueType()));
11206 SDValue LHS = DAG.getSplatVector(VT: InterVT, DL, Op: SplatVal);
11207 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: InterVT);
11208 return DAG.getSetCC(DL, VT, LHS, RHS: Zero, Cond: ISD::SETNE);
11209}
11210
11211// Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
11212// illegal (currently only vXi64 RV32).
11213// FIXME: We could also catch non-constant sign-extended i32 values and lower
11214// them to VMV_V_X_VL.
11215SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
11216 SelectionDAG &DAG) const {
11217 SDLoc DL(Op);
11218 MVT VecVT = Op.getSimpleValueType();
11219 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
11220 "Unexpected SPLAT_VECTOR_PARTS lowering");
11221
11222 assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
11223 SDValue Lo = Op.getOperand(i: 0);
11224 SDValue Hi = Op.getOperand(i: 1);
11225
11226 MVT ContainerVT = VecVT;
11227 if (VecVT.isFixedLengthVector())
11228 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11229
11230 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
11231
11232 SDValue Res =
11233 splatPartsI64WithVL(DL, VT: ContainerVT, Passthru: SDValue(), Lo, Hi, VL, DAG);
11234
11235 if (VecVT.isFixedLengthVector())
11236 Res = convertFromScalableVector(VT: VecVT, V: Res, DAG, Subtarget);
11237
11238 return Res;
11239}
11240
11241// Custom-lower extensions from mask vectors by using a vselect either with 1
11242// for zero/any-extension or -1 for sign-extension:
11243// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
11244// Note that any-extension is lowered identically to zero-extension.
11245SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
11246 int64_t ExtTrueVal) const {
11247 SDLoc DL(Op);
11248 MVT VecVT = Op.getSimpleValueType();
11249 SDValue Src = Op.getOperand(i: 0);
11250 // Only custom-lower extensions from mask types
11251 assert(Src.getValueType().isVector() &&
11252 Src.getValueType().getVectorElementType() == MVT::i1);
11253
11254 if (VecVT.isScalableVector()) {
11255 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: VecVT);
11256 SDValue SplatTrueVal = DAG.getSignedConstant(Val: ExtTrueVal, DL, VT: VecVT);
11257 if (Src.getOpcode() == ISD::XOR &&
11258 ISD::isConstantSplatVectorAllOnes(N: Src.getOperand(i: 1).getNode()))
11259 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Src.getOperand(i: 0), N2: SplatZero,
11260 N3: SplatTrueVal);
11261 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Src, N2: SplatTrueVal, N3: SplatZero);
11262 }
11263
11264 MVT ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11265 MVT I1ContainerVT =
11266 MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
11267
11268 SDValue CC = convertToScalableVector(VT: I1ContainerVT, V: Src, DAG, Subtarget);
11269
11270 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
11271
11272 MVT XLenVT = Subtarget.getXLenVT();
11273 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
11274 SDValue SplatTrueVal = DAG.getSignedConstant(Val: ExtTrueVal, DL, VT: XLenVT);
11275
11276 if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
11277 SDValue Xor = Src.getOperand(i: 0);
11278 if (Xor.getOpcode() == RISCVISD::VMXOR_VL) {
11279 SDValue ScalableOnes = Xor.getOperand(i: 1);
11280 if (ScalableOnes.getOpcode() == ISD::INSERT_SUBVECTOR &&
11281 ScalableOnes.getOperand(i: 0).isUndef() &&
11282 ISD::isConstantSplatVectorAllOnes(
11283 N: ScalableOnes.getOperand(i: 1).getNode())) {
11284 CC = Xor.getOperand(i: 0);
11285 std::swap(a&: SplatZero, b&: SplatTrueVal);
11286 }
11287 }
11288 }
11289
11290 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
11291 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatZero, N3: VL);
11292 SplatTrueVal = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
11293 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatTrueVal, N3: VL);
11294 SDValue Select =
11295 DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: CC, N2: SplatTrueVal,
11296 N3: SplatZero, N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
11297
11298 return convertFromScalableVector(VT: VecVT, V: Select, DAG, Subtarget);
11299}
11300
11301// Custom-lower truncations from vectors to mask vectors by using a mask and a
11302// setcc operation:
11303// (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
11304SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
11305 SelectionDAG &DAG) const {
11306 SDLoc DL(Op);
11307 EVT MaskVT = Op.getValueType();
11308 // Only expect to custom-lower truncations to mask types
11309 assert(MaskVT.isVectorOf(MVT::i1) &&
11310 "Unexpected type for vector mask lowering");
11311 SDValue Src = Op.getOperand(i: 0);
11312 MVT VecVT = Src.getSimpleValueType();
11313 // If this is a fixed vector, we need to convert it to a scalable vector.
11314 MVT ContainerVT = VecVT;
11315
11316 if (VecVT.isFixedLengthVector()) {
11317 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11318 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
11319 }
11320
11321 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11322
11323 SDValue SplatOne = DAG.getConstant(Val: 1, DL, VT: Subtarget.getXLenVT());
11324 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
11325
11326 SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
11327 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatOne, N3: VL);
11328 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
11329 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatZero, N3: VL);
11330
11331 MVT MaskContainerVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
11332 SDValue Trunc = DAG.getNode(Opcode: RISCVISD::AND_VL, DL, VT: ContainerVT, N1: Src, N2: SplatOne,
11333 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
11334 Trunc = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: MaskContainerVT,
11335 Ops: {Trunc, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
11336 DAG.getUNDEF(VT: MaskContainerVT), Mask, VL});
11337 if (MaskVT.isFixedLengthVector())
11338 Trunc = convertFromScalableVector(VT: MaskVT, V: Trunc, DAG, Subtarget);
11339 return Trunc;
11340}
11341
11342SDValue RISCVTargetLowering::lowerVectorTrunc(SDValue Op,
11343 SelectionDAG &DAG) const {
11344 unsigned Opc = Op.getOpcode();
11345 SDLoc DL(Op);
11346
11347 MVT VT = Op.getSimpleValueType();
11348 // Only custom-lower vector truncates
11349 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
11350
11351 // Truncates to mask types are handled differently
11352 if (VT.getVectorElementType() == MVT::i1)
11353 return lowerVectorMaskTrunc(Op, DAG);
11354
11355 // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
11356 // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
11357 // truncate by one power of two at a time.
11358 MVT DstEltVT = VT.getVectorElementType();
11359
11360 SDValue Src = Op.getOperand(i: 0);
11361 MVT SrcVT = Src.getSimpleValueType();
11362 MVT SrcEltVT = SrcVT.getVectorElementType();
11363
11364 assert(DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) &&
11365 isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
11366 "Unexpected vector truncate lowering");
11367
11368 MVT ContainerVT = SrcVT;
11369 if (SrcVT.isFixedLengthVector()) {
11370 ContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
11371 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
11372 }
11373
11374 SDValue Result = Src;
11375 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
11376
11377 unsigned NewOpc;
11378 if (Opc == ISD::TRUNCATE_SSAT_S)
11379 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
11380 else if (Opc == ISD::TRUNCATE_USAT_U)
11381 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
11382 else
11383 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
11384
11385 do {
11386 SrcEltVT = MVT::getIntegerVT(BitWidth: SrcEltVT.getSizeInBits() / 2);
11387 MVT ResultVT = ContainerVT.changeVectorElementType(EltVT: SrcEltVT);
11388 Result = DAG.getNode(Opcode: NewOpc, DL, VT: ResultVT, N1: Result, N2: Mask, N3: VL);
11389 } while (SrcEltVT != DstEltVT);
11390
11391 if (SrcVT.isFixedLengthVector())
11392 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
11393
11394 return Result;
11395}
11396
11397SDValue
11398RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(SDValue Op,
11399 SelectionDAG &DAG) const {
11400 SDLoc DL(Op);
11401 SDValue Chain = Op.getOperand(i: 0);
11402 SDValue Src = Op.getOperand(i: 1);
11403 MVT VT = Op.getSimpleValueType();
11404 MVT SrcVT = Src.getSimpleValueType();
11405 MVT ContainerVT = VT;
11406 if (VT.isFixedLengthVector()) {
11407 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
11408 ContainerVT =
11409 SrcContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType());
11410 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
11411 }
11412
11413 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
11414
11415 // RVV can only widen/truncate fp to types double/half the size as the source.
11416 if ((VT.getVectorElementType() == MVT::f64 &&
11417 (SrcVT.getVectorElementType() == MVT::f16 ||
11418 SrcVT.getVectorElementType() == MVT::bf16)) ||
11419 ((VT.getVectorElementType() == MVT::f16 ||
11420 VT.getVectorElementType() == MVT::bf16) &&
11421 SrcVT.getVectorElementType() == MVT::f64)) {
11422 // For double rounding, the intermediate rounding should be round-to-odd.
11423 unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
11424 ? RISCVISD::STRICT_FP_EXTEND_VL
11425 : RISCVISD::STRICT_VFNCVT_ROD_VL;
11426 MVT InterVT = ContainerVT.changeVectorElementType(EltVT: MVT::f32);
11427 Src = DAG.getNode(Opcode: InterConvOpc, DL, VTList: DAG.getVTList(VT1: InterVT, VT2: MVT::Other),
11428 N1: Chain, N2: Src, N3: Mask, N4: VL);
11429 Chain = Src.getValue(R: 1);
11430 }
11431
11432 unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
11433 ? RISCVISD::STRICT_FP_EXTEND_VL
11434 : RISCVISD::STRICT_FP_ROUND_VL;
11435 SDValue Res = DAG.getNode(Opcode: ConvOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
11436 N1: Chain, N2: Src, N3: Mask, N4: VL);
11437 if (VT.isFixedLengthVector()) {
11438 // StrictFP operations have two result values. Their lowered result should
11439 // have same result count.
11440 SDValue SubVec = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
11441 Res = DAG.getMergeValues(Ops: {SubVec, Res.getValue(R: 1)}, dl: DL);
11442 }
11443 return Res;
11444}
11445
11446SDValue
11447RISCVTargetLowering::lowerVectorFPExtendOrRound(SDValue Op,
11448 SelectionDAG &DAG) const {
11449 bool IsExtend = Op.getOpcode() == ISD::FP_EXTEND;
11450 // RVV can only do truncate fp to types half the size as the source. We
11451 // custom-lower f64->f16 rounds via RVV's round-to-odd float
11452 // conversion instruction.
11453 SDLoc DL(Op);
11454 MVT VT = Op.getSimpleValueType();
11455
11456 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
11457
11458 SDValue Src = Op.getOperand(i: 0);
11459 MVT SrcVT = Src.getSimpleValueType();
11460
11461 bool IsDirectExtend =
11462 IsExtend && (VT.getVectorElementType() != MVT::f64 ||
11463 (SrcVT.getVectorElementType() != MVT::f16 &&
11464 SrcVT.getVectorElementType() != MVT::bf16));
11465 bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
11466 VT.getVectorElementType() != MVT::bf16) ||
11467 SrcVT.getVectorElementType() != MVT::f64);
11468
11469 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
11470
11471 // We have regular SD node patterns for direct non-VL extends.
11472 if (VT.isScalableVector() && IsDirectConv)
11473 return Op;
11474
11475 // Prepare any fixed-length vector operands.
11476 MVT ContainerVT = VT;
11477 if (VT.isFixedLengthVector()) {
11478 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
11479 ContainerVT =
11480 SrcContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType());
11481 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
11482 }
11483
11484 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
11485
11486 unsigned ConvOpc = IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::FP_ROUND_VL;
11487
11488 if (IsDirectConv) {
11489 Src = DAG.getNode(Opcode: ConvOpc, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
11490 if (VT.isFixedLengthVector())
11491 Src = convertFromScalableVector(VT, V: Src, DAG, Subtarget);
11492 return Src;
11493 }
11494
11495 unsigned InterConvOpc =
11496 IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::VFNCVT_ROD_VL;
11497
11498 MVT InterVT = ContainerVT.changeVectorElementType(EltVT: MVT::f32);
11499 SDValue IntermediateConv =
11500 DAG.getNode(Opcode: InterConvOpc, DL, VT: InterVT, N1: Src, N2: Mask, N3: VL);
11501 SDValue Result =
11502 DAG.getNode(Opcode: ConvOpc, DL, VT: ContainerVT, N1: IntermediateConv, N2: Mask, N3: VL);
11503 if (VT.isFixedLengthVector())
11504 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
11505 return Result;
11506}
11507
11508// Given a scalable vector type and an index into it, returns the type for the
11509// smallest subvector that the index fits in. This can be used to reduce LMUL
11510// for operations like vslidedown.
11511//
11512// E.g. With Zvl128b, index 3 in a nxv4i32 fits within the first nxv2i32.
11513static std::optional<MVT>
11514getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
11515 const RISCVSubtarget &Subtarget) {
11516 assert(VecVT.isScalableVector());
11517 const unsigned EltSize = VecVT.getScalarSizeInBits();
11518 const unsigned VectorBitsMin = Subtarget.getRealMinVLen();
11519 const unsigned MinVLMAX = VectorBitsMin / EltSize;
11520 MVT SmallerVT;
11521 if (MaxIdx < MinVLMAX)
11522 SmallerVT = RISCVTargetLowering::getM1VT(VT: VecVT);
11523 else if (MaxIdx < MinVLMAX * 2)
11524 SmallerVT =
11525 RISCVTargetLowering::getM1VT(VT: VecVT).getDoubleNumVectorElementsVT();
11526 else if (MaxIdx < MinVLMAX * 4)
11527 SmallerVT = RISCVTargetLowering::getM1VT(VT: VecVT)
11528 .getDoubleNumVectorElementsVT()
11529 .getDoubleNumVectorElementsVT();
11530 if (!SmallerVT.isValid() || !VecVT.bitsGT(VT: SmallerVT))
11531 return std::nullopt;
11532 return SmallerVT;
11533}
11534
11535// Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
11536// first position of a vector, and that vector is slid up to the insert index.
11537// By limiting the active vector length to index+1 and merging with the
11538// original vector (with an undisturbed tail policy for elements >= VL), we
11539// achieve the desired result of leaving all elements untouched except the one
11540// at VL-1, which is replaced with the desired value.
11541SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
11542 SelectionDAG &DAG) const {
11543 SDLoc DL(Op);
11544 MVT VecVT = Op.getSimpleValueType();
11545 MVT XLenVT = Subtarget.getXLenVT();
11546 SDValue Vec = Op.getOperand(i: 0);
11547 SDValue Val = Op.getOperand(i: 1);
11548 MVT ValVT = Val.getSimpleValueType();
11549 SDValue Idx = Op.getOperand(i: 2);
11550
11551 if (VecVT.getVectorElementType() == MVT::i1) {
11552 // FIXME: For now we just promote to an i8 vector and insert into that,
11553 // but this is probably not optimal.
11554 MVT WideVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
11555 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Vec);
11556 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: WideVT, N1: Vec, N2: Val, N3: Idx);
11557 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Vec);
11558 }
11559
11560 if ((ValVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11561 (ValVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11562 // If we don't have vfmv.s.f for f16/bf16, use fmv.x.h first.
11563 MVT IntVT = VecVT.changeTypeToInteger();
11564 SDValue IntInsert = DAG.getNode(
11565 Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVT, N1: DAG.getBitcast(VT: IntVT, V: Vec),
11566 N2: DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Val), N3: Idx);
11567 return DAG.getBitcast(VT: VecVT, V: IntInsert);
11568 }
11569
11570 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11571 auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11572 if (!IdxC)
11573 return SDValue();
11574
11575 unsigned IdxVal = IdxC->getZExtValue();
11576 unsigned NumElts = VecVT.getVectorNumElements();
11577 MVT EltVT = VecVT.getVectorElementType();
11578
11579 if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
11580 unsigned HalfNumElts = NumElts / 2;
11581 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
11582 MVT HalfVT = Lo.getSimpleValueType();
11583 if (IdxVal < HalfNumElts) {
11584 SDValue NewLo =
11585 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: HalfVT, N1: Lo, N2: Val, N3: Idx);
11586 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: NewLo, N2: Hi);
11587 }
11588 SDValue NewHi =
11589 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: HalfVT, N1: Hi, N2: Val,
11590 N3: DAG.getVectorIdxConstant(Val: IdxVal - HalfNumElts, DL));
11591 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo, N2: NewHi);
11592 }
11593
11594 Vec = DAG.getBitcast(VT: XLenVT, V: Vec);
11595 SDValue ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Val);
11596
11597 // For 2-element vectors, BUILD_VECTOR is more efficient since it only needs
11598 // at most 2 instructions.
11599 if (NumElts == 2) {
11600 unsigned EltBits = EltVT.getSizeInBits();
11601 SDValue Elt0, Elt1;
11602 if (IdxVal == 0) {
11603 Elt0 = ExtVal;
11604 Elt1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Vec,
11605 N2: DAG.getConstant(Val: EltBits, DL, VT: XLenVT));
11606 } else {
11607 Elt0 = Vec;
11608 Elt1 = ExtVal;
11609 }
11610 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: VecVT, N1: Elt0, N2: Elt1);
11611 }
11612
11613 // For 4/8-element vectors, use MVM(or MERGE) instruction which does bitwise
11614 // select: rd = (~mask & rd) | (mask & rs1).
11615 // This generates: slli + lui/li + mvm
11616 if (NumElts == 4 || NumElts == 8) {
11617 unsigned EltBits = EltVT.getSizeInBits();
11618 unsigned ShiftAmt = IdxVal * EltBits;
11619 uint64_t PosMask = ((1ULL << EltBits) - 1) << ShiftAmt;
11620
11621 SDValue ShiftedVal = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: ExtVal,
11622 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: XLenVT));
11623 SDValue Mask = DAG.getConstant(Val: PosMask, DL, VT: XLenVT);
11624 SDValue Result =
11625 DAG.getNode(Opcode: RISCVISD::MERGE, DL, VT: XLenVT, N1: Mask, N2: Vec, N3: ShiftedVal);
11626 return DAG.getBitcast(VT: VecVT, V: Result);
11627 }
11628
11629 return SDValue();
11630 }
11631
11632 MVT ContainerVT = VecVT;
11633 // If the operand is a fixed-length vector, convert to a scalable one.
11634 if (VecVT.isFixedLengthVector()) {
11635 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11636 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11637 }
11638
11639 // If we know the index we're going to insert at, we can shrink Vec so that
11640 // we're performing the scalar inserts and slideup on a smaller LMUL.
11641 SDValue OrigVec = Vec;
11642 std::optional<unsigned> AlignedIdx;
11643 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx)) {
11644 const unsigned OrigIdx = IdxC->getZExtValue();
11645 // Do we know an upper bound on LMUL?
11646 if (auto ShrunkVT = getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: OrigIdx,
11647 DL, DAG, Subtarget)) {
11648 ContainerVT = *ShrunkVT;
11649 AlignedIdx = 0;
11650 }
11651
11652 // If we're compiling for an exact VLEN value, we can always perform
11653 // the insert in m1 as we can determine the register corresponding to
11654 // the index in the register group.
11655 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
11656 if (auto VLEN = Subtarget.getRealVLen(); VLEN && ContainerVT.bitsGT(VT: M1VT)) {
11657 EVT ElemVT = VecVT.getVectorElementType();
11658 unsigned ElemsPerVReg = *VLEN / ElemVT.getFixedSizeInBits();
11659 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11660 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11661 AlignedIdx = SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11662 Idx = DAG.getVectorIdxConstant(Val: RemIdx, DL);
11663 ContainerVT = M1VT;
11664 }
11665
11666 if (AlignedIdx)
11667 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: *AlignedIdx);
11668 }
11669
11670 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
11671 // Even i64-element vectors on RV32 can be lowered without scalar
11672 // legalization if the most-significant 32 bits of the value are not affected
11673 // by the sign-extension of the lower 32 bits. This applies to i32 constants
11674 // and sign_extend of i32 values.
11675 if (!IsLegalInsert) {
11676 if (isa<ConstantSDNode>(Val)) {
11677 const auto *CVal = cast<ConstantSDNode>(Val);
11678 if (isInt<32>(x: CVal->getSExtValue())) {
11679 IsLegalInsert = true;
11680 Val = DAG.getSignedConstant(Val: CVal->getSExtValue(), DL, VT: MVT::i32);
11681 }
11682 } else if (Val.getOpcode() == ISD::SIGN_EXTEND &&
11683 Val.getOperand(i: 0).getValueType() == MVT::i32) {
11684 IsLegalInsert = true;
11685 Val = Val.getOperand(i: 0);
11686 }
11687 }
11688
11689 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11690
11691 SDValue ValInVec;
11692
11693 if (IsLegalInsert) {
11694 unsigned Opc =
11695 VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
11696 if (isNullConstant(V: Idx)) {
11697 if (!VecVT.isFloatingPoint())
11698 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Val);
11699 Vec = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: Vec, N2: Val, N3: VL);
11700
11701 if (AlignedIdx)
11702 Vec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Vec, Idx: *AlignedIdx);
11703 if (!VecVT.isFixedLengthVector())
11704 return Vec;
11705 return convertFromScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
11706 }
11707
11708 ValInVec = lowerScalarInsert(Scalar: Val, VL, VT: ContainerVT, DL, DAG, Subtarget);
11709 } else {
11710 // On RV32, i64-element vectors must be specially handled to place the
11711 // value at element 0, by using two vslide1down instructions in sequence on
11712 // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
11713 // this.
11714 SDValue ValLo, ValHi;
11715 std::tie(args&: ValLo, args&: ValHi) = DAG.SplitScalar(N: Val, DL, LoVT: MVT::i32, HiVT: MVT::i32);
11716 MVT I32ContainerVT =
11717 MVT::getVectorVT(VT: MVT::i32, EC: ContainerVT.getVectorElementCount() * 2);
11718 SDValue I32Mask =
11719 getDefaultScalableVLOps(VecVT: I32ContainerVT, DL, DAG, Subtarget).first;
11720 // Limit the active VL to two.
11721 SDValue InsertI64VL = DAG.getConstant(Val: 2, DL, VT: XLenVT);
11722 // If the Idx is 0 we can insert directly into the vector.
11723 if (isNullConstant(V: Idx)) {
11724 // First slide in the lo value, then the hi in above it. We use slide1down
11725 // to avoid the register group overlap constraint of vslide1up.
11726 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11727 N1: Vec, N2: Vec, N3: ValLo, N4: I32Mask, N5: InsertI64VL);
11728 // If the source vector is undef don't pass along the tail elements from
11729 // the previous slide1down.
11730 SDValue Tail = Vec.isUndef() ? Vec : ValInVec;
11731 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11732 N1: Tail, N2: ValInVec, N3: ValHi, N4: I32Mask, N5: InsertI64VL);
11733 // Bitcast back to the right container type.
11734 ValInVec = DAG.getBitcast(VT: ContainerVT, V: ValInVec);
11735
11736 if (AlignedIdx)
11737 ValInVec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: ValInVec, Idx: *AlignedIdx);
11738 if (!VecVT.isFixedLengthVector())
11739 return ValInVec;
11740 return convertFromScalableVector(VT: VecVT, V: ValInVec, DAG, Subtarget);
11741 }
11742
11743 // First slide in the lo value, then the hi in above it. We use slide1down
11744 // to avoid the register group overlap constraint of vslide1up.
11745 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11746 N1: DAG.getUNDEF(VT: I32ContainerVT),
11747 N2: DAG.getUNDEF(VT: I32ContainerVT), N3: ValLo,
11748 N4: I32Mask, N5: InsertI64VL);
11749 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11750 N1: DAG.getUNDEF(VT: I32ContainerVT), N2: ValInVec, N3: ValHi,
11751 N4: I32Mask, N5: InsertI64VL);
11752 // Bitcast back to the right container type.
11753 ValInVec = DAG.getBitcast(VT: ContainerVT, V: ValInVec);
11754 }
11755
11756 // Now that the value is in a vector, slide it into position.
11757 SDValue InsertVL =
11758 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: Idx, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11759
11760 // Use tail agnostic policy if Idx is the last index of Vec.
11761 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
11762 if (VecVT.isFixedLengthVector() && isa<ConstantSDNode>(Val: Idx) &&
11763 Idx->getAsZExtVal() + 1 == VecVT.getVectorNumElements())
11764 Policy = RISCVVType::TAIL_AGNOSTIC;
11765 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Vec, Op: ValInVec,
11766 Offset: Idx, Mask, VL: InsertVL, Policy);
11767
11768 if (AlignedIdx)
11769 Slideup = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Slideup, Idx: *AlignedIdx);
11770 if (!VecVT.isFixedLengthVector())
11771 return Slideup;
11772 return convertFromScalableVector(VT: VecVT, V: Slideup, DAG, Subtarget);
11773}
11774
11775// Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
11776// extract the first element: (extractelt (slidedown vec, idx), 0). For integer
11777// types this is done using VMV_X_S to allow us to glean information about the
11778// sign bits of the result.
11779SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
11780 SelectionDAG &DAG) const {
11781 SDLoc DL(Op);
11782 SDValue Idx = Op.getOperand(i: 1);
11783 SDValue Vec = Op.getOperand(i: 0);
11784 EVT EltVT = Op.getValueType();
11785 MVT VecVT = Vec.getSimpleValueType();
11786 MVT XLenVT = Subtarget.getXLenVT();
11787
11788 if (VecVT.getVectorElementType() == MVT::i1) {
11789 // Use vfirst.m to extract the first bit.
11790 if (isNullConstant(V: Idx)) {
11791 MVT ContainerVT = VecVT;
11792 if (VecVT.isFixedLengthVector()) {
11793 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11794 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11795 }
11796 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11797 SDValue Vfirst =
11798 DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
11799 SDValue Res = DAG.getSetCC(DL, VT: XLenVT, LHS: Vfirst,
11800 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
11801 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
11802 }
11803 if (VecVT.isFixedLengthVector()) {
11804 unsigned NumElts = VecVT.getVectorNumElements();
11805 if (NumElts >= 8) {
11806 MVT WideEltVT;
11807 unsigned WidenVecLen;
11808 SDValue ExtractElementIdx;
11809 SDValue ExtractBitIdx;
11810 unsigned MaxEEW = Subtarget.getELen();
11811 MVT LargestEltVT = MVT::getIntegerVT(
11812 BitWidth: std::min(a: MaxEEW, b: unsigned(XLenVT.getSizeInBits())));
11813 if (NumElts <= LargestEltVT.getSizeInBits()) {
11814 assert(isPowerOf2_32(NumElts) &&
11815 "the number of elements should be power of 2");
11816 WideEltVT = MVT::getIntegerVT(BitWidth: NumElts);
11817 WidenVecLen = 1;
11818 ExtractElementIdx = DAG.getConstant(Val: 0, DL, VT: XLenVT);
11819 ExtractBitIdx = Idx;
11820 } else {
11821 WideEltVT = LargestEltVT;
11822 WidenVecLen = NumElts / WideEltVT.getSizeInBits();
11823 // extract element index = index / element width
11824 ExtractElementIdx = DAG.getNode(
11825 Opcode: ISD::SRL, DL, VT: XLenVT, N1: Idx,
11826 N2: DAG.getConstant(Val: Log2_64(Value: WideEltVT.getSizeInBits()), DL, VT: XLenVT));
11827 // mask bit index = index % element width
11828 ExtractBitIdx = DAG.getNode(
11829 Opcode: ISD::AND, DL, VT: XLenVT, N1: Idx,
11830 N2: DAG.getConstant(Val: WideEltVT.getSizeInBits() - 1, DL, VT: XLenVT));
11831 }
11832 MVT WideVT = MVT::getVectorVT(VT: WideEltVT, NumElements: WidenVecLen);
11833 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Vec);
11834 SDValue ExtractElt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: XLenVT,
11835 N1: Vec, N2: ExtractElementIdx);
11836 // Extract the bit from GPR.
11837 SDValue ShiftRight =
11838 DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: ExtractElt, N2: ExtractBitIdx);
11839 SDValue Res = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: ShiftRight,
11840 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11841 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
11842 }
11843 }
11844 // Otherwise, promote to an i8 vector and extract from that.
11845 MVT WideVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
11846 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Vec);
11847 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Vec, N2: Idx);
11848 }
11849
11850 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11851 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11852 // If we don't have vfmv.f.s for f16/bf16, extract to a gpr then use fmv.h.x
11853 MVT IntVT = VecVT.changeTypeToInteger();
11854 SDValue IntVec = DAG.getBitcast(VT: IntVT, V: Vec);
11855 SDValue IntExtract =
11856 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: XLenVT, N1: IntVec, N2: Idx);
11857 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT: EltVT, Operand: IntExtract);
11858 }
11859
11860 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11861 if (VecVT != MVT::v4i16 && VecVT != MVT::v2i16 && VecVT != MVT::v8i8 &&
11862 VecVT != MVT::v4i8 && VecVT != MVT::v2i32)
11863 return SDValue();
11864
11865 if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
11866 auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11867 if (!IdxC)
11868 return SDValue();
11869 unsigned IdxVal = IdxC->getZExtValue();
11870 unsigned HalfNumElts = VecVT.getVectorNumElements() / 2;
11871 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
11872 if (IdxVal < HalfNumElts)
11873 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Lo, N2: Idx);
11874 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Hi,
11875 N2: DAG.getVectorIdxConstant(Val: IdxVal - HalfNumElts, DL));
11876 }
11877
11878 SDValue Extracted = DAG.getBitcast(VT: XLenVT, V: Vec);
11879 unsigned ElemWidth = VecVT.getVectorElementType().getSizeInBits();
11880 SDValue Shamt = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Idx,
11881 N2: DAG.getConstant(Val: ElemWidth, DL, VT: XLenVT));
11882 return DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Extracted, N2: Shamt);
11883 }
11884
11885 // If this is a fixed vector, we need to convert it to a scalable vector.
11886 MVT ContainerVT = VecVT;
11887 if (VecVT.isFixedLengthVector()) {
11888 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11889 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11890 }
11891
11892 // If we're compiling for an exact VLEN value and we have a known
11893 // constant index, we can always perform the extract in m1 (or
11894 // smaller) as we can determine the register corresponding to
11895 // the index in the register group.
11896 const auto VLen = Subtarget.getRealVLen();
11897 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11898 IdxC && VLen && VecVT.getSizeInBits().getKnownMinValue() > *VLen) {
11899 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
11900 unsigned OrigIdx = IdxC->getZExtValue();
11901 EVT ElemVT = VecVT.getVectorElementType();
11902 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
11903 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11904 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11905 unsigned ExtractIdx =
11906 SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11907 Vec = DAG.getExtractSubvector(DL, VT: M1VT, Vec, Idx: ExtractIdx);
11908 Idx = DAG.getVectorIdxConstant(Val: RemIdx, DL);
11909 ContainerVT = M1VT;
11910 }
11911
11912 // Reduce the LMUL of our slidedown and vmv.x.s to the smallest LMUL which
11913 // contains our index.
11914 std::optional<uint64_t> MaxIdx;
11915 if (VecVT.isFixedLengthVector())
11916 MaxIdx = VecVT.getVectorNumElements() - 1;
11917 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx))
11918 MaxIdx = IdxC->getZExtValue();
11919 if (MaxIdx) {
11920 if (auto SmallerVT =
11921 getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: *MaxIdx, DL, DAG, Subtarget)) {
11922 ContainerVT = *SmallerVT;
11923 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: 0);
11924 }
11925 }
11926
11927 // If after narrowing, the required slide is still greater than LMUL2,
11928 // fallback to generic expansion and go through the stack. This is done
11929 // for a subtle reason: extracting *all* elements out of a vector is
11930 // widely expected to be linear in vector size, but because vslidedown
11931 // is linear in LMUL, performing N extracts using vslidedown becomes
11932 // O(n^2) / (VLEN/ETYPE) work. On the surface, going through the stack
11933 // seems to have the same problem (the store is linear in LMUL), but the
11934 // generic expansion *memoizes* the store, and thus for many extracts of
11935 // the same vector we end up with one store and a bunch of loads.
11936 // TODO: We don't have the same code for insert_vector_elt because we
11937 // have BUILD_VECTOR and handle the degenerate case there. Should we
11938 // consider adding an inverse BUILD_VECTOR node?
11939 MVT LMUL2VT =
11940 RISCVTargetLowering::getM1VT(VT: ContainerVT).getDoubleNumVectorElementsVT();
11941 if (ContainerVT.bitsGT(VT: LMUL2VT) && VecVT.isFixedLengthVector())
11942 return SDValue();
11943
11944 // If the index is 0, the vector is already in the right position.
11945 if (!isNullConstant(V: Idx)) {
11946 // Use a VL of 1 to avoid processing more elements than we need.
11947 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT, DL, DAG, Subtarget);
11948 Vec = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
11949 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: Idx, Mask, VL);
11950 }
11951
11952 if (!EltVT.isInteger()) {
11953 // Floating-point extracts are handled in TableGen.
11954 return DAG.getExtractVectorElt(DL, VT: EltVT, Vec, Idx: 0);
11955 }
11956
11957 SDValue Elt0 = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
11958 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Elt0);
11959}
11960
11961// Some RVV intrinsics may claim that they want an integer operand to be
11962// promoted or expanded.
11963static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
11964 const RISCVSubtarget &Subtarget) {
11965 assert((Op.getOpcode() == ISD::INTRINSIC_VOID ||
11966 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
11967 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
11968 "Unexpected opcode");
11969
11970 if (!Subtarget.hasVInstructions())
11971 return SDValue();
11972
11973 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11974 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11975 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
11976
11977 SDLoc DL(Op);
11978
11979 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11980 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
11981 if (!II || !II->hasScalarOperand())
11982 return SDValue();
11983
11984 unsigned SplatOp = II->ScalarOperand + 1 + HasChain;
11985 assert(SplatOp < Op.getNumOperands());
11986
11987 SmallVector<SDValue, 8> Operands(Op->ops());
11988 SDValue &ScalarOp = Operands[SplatOp];
11989 MVT OpVT = ScalarOp.getSimpleValueType();
11990 MVT XLenVT = Subtarget.getXLenVT();
11991
11992 // If this isn't a scalar, or its type is XLenVT we're done.
11993 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11994 return SDValue();
11995
11996 // Simplest case is that the operand needs to be promoted to XLenVT.
11997 if (OpVT.bitsLT(VT: XLenVT)) {
11998 // If the operand is a constant, sign extend to increase our chances
11999 // of being able to use a .vi instruction. ANY_EXTEND would become a
12000 // a zero extend and the simm5 check in isel would fail.
12001 // FIXME: Should we ignore the upper bits in isel instead?
12002 unsigned ExtOpc =
12003 isa<ConstantSDNode>(Val: ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
12004 ScalarOp = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: ScalarOp);
12005 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
12006 }
12007
12008 // Use the previous operand to get the vXi64 VT. The result might be a mask
12009 // VT for compares. Using the previous operand assumes that the previous
12010 // operand will never have a smaller element size than a scalar operand and
12011 // that a widening operation never uses SEW=64.
12012 // NOTE: If this fails the below assert, we can probably just find the
12013 // element count from any operand or result and use it to construct the VT.
12014 assert(II->ScalarOperand > 0 && "Unexpected splat operand!");
12015 MVT VT = Op.getOperand(i: SplatOp - 1).getSimpleValueType();
12016
12017 // The more complex case is when the scalar is larger than XLenVT.
12018 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
12019 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
12020
12021 // If this is a sign-extended 32-bit value, we can truncate it and rely on the
12022 // instruction to sign-extend since SEW>XLEN.
12023 if (DAG.ComputeNumSignBits(Op: ScalarOp) > 32) {
12024 ScalarOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: ScalarOp);
12025 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
12026 }
12027
12028 switch (IntNo) {
12029 case Intrinsic::riscv_vslide1up:
12030 case Intrinsic::riscv_vslide1down:
12031 case Intrinsic::riscv_vslide1up_mask:
12032 case Intrinsic::riscv_vslide1down_mask: {
12033 // We need to special case these when the scalar is larger than XLen.
12034 unsigned NumOps = Op.getNumOperands();
12035 bool IsMasked = NumOps == 7;
12036
12037 // Convert the vector source to the equivalent nxvXi32 vector.
12038 MVT I32VT = MVT::getVectorVT(VT: MVT::i32, EC: VT.getVectorElementCount() * 2);
12039 SDValue Vec = DAG.getBitcast(VT: I32VT, V: Operands[2]);
12040 SDValue ScalarLo, ScalarHi;
12041 std::tie(args&: ScalarLo, args&: ScalarHi) =
12042 DAG.SplitScalar(N: ScalarOp, DL, LoVT: MVT::i32, HiVT: MVT::i32);
12043
12044 // Double the VL since we halved SEW.
12045 SDValue AVL = getVLOperand(Op);
12046 SDValue I32VL;
12047
12048 // Optimize for constant AVL
12049 if (isa<ConstantSDNode>(Val: AVL)) {
12050 const auto [MinVLMAX, MaxVLMAX] =
12051 RISCVTargetLowering::computeVLMAXBounds(VecVT: VT, Subtarget);
12052
12053 uint64_t AVLInt = AVL->getAsZExtVal();
12054 if (AVLInt <= MinVLMAX) {
12055 I32VL = DAG.getConstant(Val: 2 * AVLInt, DL, VT: XLenVT);
12056 } else if (AVLInt >= 2 * MaxVLMAX) {
12057 // Just set vl to VLMAX in this situation
12058 I32VL = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
12059 } else {
12060 // For AVL between (MinVLMAX, 2 * MaxVLMAX), the actual working vl
12061 // is related to the hardware implementation.
12062 // So let the following code handle
12063 }
12064 }
12065 if (!I32VL) {
12066 RISCVVType::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT);
12067 SDValue LMUL = DAG.getConstant(Val: Lmul, DL, VT: XLenVT);
12068 unsigned Sew = RISCVVType::encodeSEW(SEW: VT.getScalarSizeInBits());
12069 SDValue SEW = DAG.getConstant(Val: Sew, DL, VT: XLenVT);
12070 SDValue SETVL =
12071 DAG.getTargetConstant(Val: Intrinsic::riscv_vsetvli, DL, VT: MVT::i32);
12072 // Using vsetvli instruction to get actually used length which related to
12073 // the hardware implementation
12074 SDValue VL = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: XLenVT, N1: SETVL, N2: AVL,
12075 N3: SEW, N4: LMUL);
12076 I32VL =
12077 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VL, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
12078 }
12079
12080 SDValue I32Mask = getAllOnesMask(VecVT: I32VT, VL: I32VL, DL, DAG);
12081
12082 // Shift the two scalar parts in using SEW=32 slide1up/slide1down
12083 // instructions.
12084 SDValue Passthru;
12085 if (IsMasked)
12086 Passthru = DAG.getUNDEF(VT: I32VT);
12087 else
12088 Passthru = DAG.getBitcast(VT: I32VT, V: Operands[1]);
12089
12090 if (IntNo == Intrinsic::riscv_vslide1up ||
12091 IntNo == Intrinsic::riscv_vslide1up_mask) {
12092 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1UP_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
12093 N3: ScalarHi, N4: I32Mask, N5: I32VL);
12094 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1UP_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
12095 N3: ScalarLo, N4: I32Mask, N5: I32VL);
12096 } else {
12097 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
12098 N3: ScalarLo, N4: I32Mask, N5: I32VL);
12099 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
12100 N3: ScalarHi, N4: I32Mask, N5: I32VL);
12101 }
12102
12103 // Convert back to nxvXi64.
12104 Vec = DAG.getBitcast(VT, V: Vec);
12105
12106 if (!IsMasked)
12107 return Vec;
12108 // Apply mask after the operation.
12109 SDValue Mask = Operands[NumOps - 3];
12110 SDValue MaskedOff = Operands[1];
12111 // Assume Policy operand is the last operand.
12112 uint64_t Policy = Operands[NumOps - 1]->getAsZExtVal();
12113 // We don't need to select maskedoff if it's undef.
12114 if (MaskedOff.isUndef())
12115 return Vec;
12116 // TAMU
12117 if (Policy == RISCVVType::TAIL_AGNOSTIC)
12118 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: Mask, N2: Vec, N3: MaskedOff,
12119 N4: DAG.getUNDEF(VT), N5: AVL);
12120 // TUMA or TUMU: Currently we always emit tumu policy regardless of tuma.
12121 // It's fine because vmerge does not care mask policy.
12122 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: Mask, N2: Vec, N3: MaskedOff,
12123 N4: MaskedOff, N5: AVL);
12124 }
12125 }
12126
12127 // We need to convert the scalar to a splat vector.
12128 SDValue VL = getVLOperand(Op);
12129 assert(VL.getValueType() == XLenVT);
12130 ScalarOp = splatSplitI64WithVL(DL, VT, Passthru: SDValue(), Scalar: ScalarOp, VL, DAG);
12131 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
12132}
12133
12134// Lower the llvm.get.vector.length intrinsic to vsetvli. We only support
12135// scalable vector llvm.get.vector.length for now.
12136//
12137// We need to convert from a scalable VF to a vsetvli with VLMax equal to
12138// (vscale * VF). The vscale and VF are independent of element width. We use
12139// SEW=8 for the vsetvli because it is the only element width that supports all
12140// fractional LMULs. The LMUL is chosen so that with SEW=8 the VLMax is
12141// (vscale * VF). Where vscale is defined as VLEN/RVVBitsPerBlock. The
12142// InsertVSETVLI pass can fix up the vtype of the vsetvli if a different
12143// SEW and LMUL are better for the surrounding vector instructions.
12144static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
12145 const RISCVSubtarget &Subtarget) {
12146 MVT XLenVT = Subtarget.getXLenVT();
12147
12148 // The smallest LMUL is only valid for the smallest element width.
12149 const unsigned ElementWidth = 8;
12150
12151 // Determine the VF that corresponds to LMUL 1 for ElementWidth.
12152 unsigned LMul1VF = RISCV::RVVBitsPerBlock / ElementWidth;
12153 // We don't support VF==1 with ELEN==32.
12154 [[maybe_unused]] unsigned MinVF =
12155 RISCV::RVVBitsPerBlock / Subtarget.getELen();
12156
12157 [[maybe_unused]] unsigned VF = N->getConstantOperandVal(Num: 2);
12158 assert(VF >= MinVF && VF <= (LMul1VF * 8) && isPowerOf2_32(VF) &&
12159 "Unexpected VF");
12160
12161 bool Fractional = VF < LMul1VF;
12162 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
12163 unsigned VLMUL = (unsigned)RISCVVType::encodeLMUL(LMUL: LMulVal, Fractional);
12164 unsigned VSEW = RISCVVType::encodeSEW(SEW: ElementWidth);
12165
12166 SDLoc DL(N);
12167
12168 SDValue LMul = DAG.getTargetConstant(Val: VLMUL, DL, VT: XLenVT);
12169 SDValue Sew = DAG.getTargetConstant(Val: VSEW, DL, VT: XLenVT);
12170
12171 SDValue AVL = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 1));
12172
12173 SDValue ID = DAG.getTargetConstant(Val: Intrinsic::riscv_vsetvli, DL, VT: XLenVT);
12174 SDValue Res =
12175 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: XLenVT, N1: ID, N2: AVL, N3: Sew, N4: LMul);
12176 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: Res);
12177}
12178
12179static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
12180 const RISCVSubtarget &Subtarget) {
12181 SDValue Op0 = Op.getOperand(i: 0);
12182 MVT OpVT = Op0.getSimpleValueType();
12183 MVT ContainerVT = OpVT;
12184 if (OpVT.isFixedLengthVector()) {
12185 ContainerVT = getContainerForFixedLengthVector(VT: OpVT, Subtarget);
12186 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
12187 }
12188 MVT XLenVT = Subtarget.getXLenVT();
12189 SDLoc DL(Op);
12190 auto [Mask, VL] = getDefaultVLOps(VecVT: OpVT, ContainerVT, DL, DAG, Subtarget);
12191 SDValue Res = DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Op0, N2: Mask, N3: VL);
12192 if (Op.getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
12193 return Res;
12194
12195 // Convert -1 to VL.
12196 SDValue Setcc =
12197 DAG.getSetCC(DL, VT: XLenVT, LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETLT);
12198 VL = DAG.getElementCount(DL, VT: XLenVT, EC: OpVT.getVectorElementCount());
12199 return DAG.getSelect(DL, VT: XLenVT, Cond: Setcc, LHS: VL, RHS: Res);
12200}
12201
12202static SDValue lowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG,
12203 const RISCVSubtarget &Subtarget) {
12204 assert(Op.getOpcode() == ISD::CONVERT_FROM_ARBITRARY_FP);
12205 const uint64_t SemEnum = Op->getConstantOperandVal(Num: 1);
12206 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
12207
12208 if (Sem != APFloatBase::S_Float8E5M2)
12209 return SDValue();
12210
12211 SDValue Src = Op.getOperand(i: 0);
12212 EVT SrcEVT = Src.getValueType();
12213 if (!SrcEVT.isSimple() || SrcEVT.getVectorElementType() != MVT::i8)
12214 return SDValue();
12215 MVT SrcVT = SrcEVT.getSimpleVT();
12216 MVT DstVT = Op.getSimpleValueType();
12217 assert(DstVT.getVectorElementType() == MVT::bf16);
12218 SDLoc DL(Op);
12219
12220 MVT SrcContainerVT = SrcVT;
12221 MVT DstContainerVT = DstVT;
12222 if (SrcVT.isFixedLengthVector()) {
12223 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
12224 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
12225 DstContainerVT = getContainerForFixedLengthVector(VT: DstVT, Subtarget);
12226 }
12227 SDValue VL =
12228 getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget).second;
12229
12230 SDValue Ops[] = {DAG.getTargetConstant(Val: Intrinsic::riscv_vfwcvt_f_f_v_alt, DL,
12231 VT: Subtarget.getXLenVT()),
12232 DAG.getPOISON(VT: DstContainerVT), // Passthru
12233 Src, VL};
12234 SDValue NewVal =
12235 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: DstContainerVT, Ops);
12236 if (SrcVT.isFixedLengthVector())
12237 NewVal = convertFromScalableVector(VT: DstVT, V: NewVal, DAG, Subtarget);
12238 return NewVal;
12239}
12240
12241static inline void promoteVCIXScalar(SDValue Op,
12242 MutableArrayRef<SDValue> Operands,
12243 SelectionDAG &DAG) {
12244 const RISCVSubtarget &Subtarget =
12245 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
12246
12247 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
12248 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
12249 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
12250 SDLoc DL(Op);
12251
12252 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
12253 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
12254 if (!II || !II->hasScalarOperand())
12255 return;
12256
12257 unsigned SplatOp = II->ScalarOperand + 1;
12258 assert(SplatOp < Op.getNumOperands());
12259
12260 SDValue &ScalarOp = Operands[SplatOp];
12261 MVT OpVT = ScalarOp.getSimpleValueType();
12262 MVT XLenVT = Subtarget.getXLenVT();
12263
12264 // The code below is partially copied from lowerVectorIntrinsicScalars.
12265 // If this isn't a scalar, or its type is XLenVT we're done.
12266 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
12267 return;
12268
12269 // Manually emit promote operation for scalar operation.
12270 if (OpVT.bitsLT(VT: XLenVT)) {
12271 unsigned ExtOpc =
12272 isa<ConstantSDNode>(Val: ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
12273 ScalarOp = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: ScalarOp);
12274 }
12275}
12276
12277static void processVCIXOperands(SDValue OrigOp,
12278 MutableArrayRef<SDValue> Operands,
12279 SelectionDAG &DAG) {
12280 promoteVCIXScalar(Op: OrigOp, Operands, DAG);
12281 const RISCVSubtarget &Subtarget =
12282 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
12283 for (SDValue &V : Operands) {
12284 EVT ValType = V.getValueType();
12285 if (ValType.isVector() && ValType.isFloatingPoint()) {
12286 MVT InterimIVT =
12287 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ValType.getScalarSizeInBits()),
12288 EC: ValType.getVectorElementCount());
12289 V = DAG.getBitcast(VT: InterimIVT, V);
12290 }
12291 if (ValType.isFixedLengthVector()) {
12292 MVT OpContainerVT =
12293 getContainerForFixedLengthVector(VT: V.getSimpleValueType(), Subtarget);
12294 V = convertToScalableVector(VT: OpContainerVT, V, DAG, Subtarget);
12295 }
12296 }
12297}
12298
12299// LMUL * VLEN should be greater than or equal to EGS * SEW
12300static inline bool isValidEGW(int EGS, EVT VT,
12301 const RISCVSubtarget &Subtarget) {
12302 return (Subtarget.getRealMinVLen() *
12303 VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
12304 EGS * VT.getScalarSizeInBits();
12305}
12306
12307static unsigned getRVPShiftOpcode(Intrinsic::ID IntNo) {
12308 switch (IntNo) {
12309 default:
12310 llvm_unreachable(
12311 "Unexpected RISC-V packed saturating and rounding shift intrinsic");
12312 case Intrinsic::riscv_pssha:
12313 return RISCVISD::PSSHA;
12314 case Intrinsic::riscv_psshar:
12315 return RISCVISD::PSSHAR;
12316 case Intrinsic::riscv_psshl:
12317 return RISCVISD::PSSHL;
12318 case Intrinsic::riscv_psshlr:
12319 return RISCVISD::PSSHLR;
12320 }
12321}
12322
12323static SDValue lowerPZExt(SDValue Src, const SDLoc &DL, SelectionDAG &DAG,
12324 const RISCVSubtarget &Subtarget) {
12325 MVT VT = Src.getSimpleValueType();
12326 MVT PPairVT =
12327 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits() / 2),
12328 NumElements: VT.getVectorNumElements() * 2);
12329 Src = DAG.getBitcast(VT: PPairVT, V: Src);
12330 SDValue Zero = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: PPairVT,
12331 Operand: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
12332 SDValue Res = DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: PPairVT, N1: Src, N2: Zero);
12333 return DAG.getBitcast(VT, V: Res);
12334}
12335
12336static unsigned getRVPMulHighOpcode(unsigned IntNo) {
12337 switch (IntNo) {
12338 default:
12339 llvm_unreachable("Unexpected RISC-V packed multiply high intrinsic");
12340 case Intrinsic::riscv_pmulh:
12341 return ISD::MULHS;
12342 case Intrinsic::riscv_pmulhr:
12343 return RISCVISD::MULHR;
12344 case Intrinsic::riscv_pmulhu:
12345 return ISD::MULHU;
12346 case Intrinsic::riscv_pmulhru:
12347 return RISCVISD::MULHRU;
12348 case Intrinsic::riscv_pmulhsu:
12349 return RISCVISD::MULHSU;
12350 case Intrinsic::riscv_pmulhrsu:
12351 return RISCVISD::MULHRSU;
12352 }
12353}
12354
12355static unsigned getRVPMulHighAccumulateOpcode(unsigned IntNo) {
12356 switch (IntNo) {
12357 default:
12358 llvm_unreachable(
12359 "Unexpected RISC-V packed multiply high accumulate intrinsic");
12360 case Intrinsic::riscv_pmhacc:
12361 return RISCVISD::MHACC;
12362 case Intrinsic::riscv_pmhracc:
12363 return RISCVISD::MHRACC;
12364 case Intrinsic::riscv_pmhaccu:
12365 return RISCVISD::MHACCU;
12366 case Intrinsic::riscv_pmhraccu:
12367 return RISCVISD::MHRACCU;
12368 case Intrinsic::riscv_pmhaccsu:
12369 return RISCVISD::MHACCSU;
12370 case Intrinsic::riscv_pmhraccsu:
12371 return RISCVISD::MHRACCSU;
12372 }
12373}
12374
12375static unsigned getRVPQFormatAccScalarOpcode(Intrinsic::ID IntNo) {
12376 switch (IntNo) {
12377 default:
12378 llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
12379 case Intrinsic::riscv_mqacc_00:
12380 case Intrinsic::riscv_pmqacc_h00:
12381 return RISCVISD::MQACC_00;
12382 case Intrinsic::riscv_mqacc_01:
12383 case Intrinsic::riscv_pmqacc_h01:
12384 return RISCVISD::MQACC_01;
12385 case Intrinsic::riscv_mqacc_11:
12386 case Intrinsic::riscv_pmqacc_h11:
12387 return RISCVISD::MQACC_11;
12388 case Intrinsic::riscv_mqracc_00:
12389 case Intrinsic::riscv_pmqracc_h00:
12390 return RISCVISD::MQRACC_00;
12391 case Intrinsic::riscv_mqracc_01:
12392 case Intrinsic::riscv_pmqracc_h01:
12393 return RISCVISD::MQRACC_01;
12394 case Intrinsic::riscv_mqracc_11:
12395 case Intrinsic::riscv_pmqracc_h11:
12396 return RISCVISD::MQRACC_11;
12397 }
12398}
12399
12400static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
12401 switch (IntNo) {
12402 default:
12403 llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
12404 case Intrinsic::riscv_mqacc_00:
12405 case Intrinsic::riscv_pmqacc_h00:
12406 return RISCVISD::PMQACC_W_H00;
12407 case Intrinsic::riscv_mqacc_01:
12408 case Intrinsic::riscv_pmqacc_h01:
12409 return RISCVISD::PMQACC_W_H01;
12410 case Intrinsic::riscv_mqacc_11:
12411 case Intrinsic::riscv_pmqacc_h11:
12412 return RISCVISD::PMQACC_W_H11;
12413 case Intrinsic::riscv_mqracc_00:
12414 case Intrinsic::riscv_pmqracc_h00:
12415 return RISCVISD::PMQRACC_W_H00;
12416 case Intrinsic::riscv_mqracc_01:
12417 case Intrinsic::riscv_pmqracc_h01:
12418 return RISCVISD::PMQRACC_W_H01;
12419 case Intrinsic::riscv_mqracc_11:
12420 case Intrinsic::riscv_pmqracc_h11:
12421 return RISCVISD::PMQRACC_W_H11;
12422 }
12423}
12424
12425static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
12426 switch (IntNo) {
12427 default:
12428 llvm_unreachable("Unexpected RISC-V packed horizontal multiply intrinsic");
12429 case Intrinsic::riscv_pm4add:
12430 return RISCVISD::PM4ADD;
12431 case Intrinsic::riscv_pm2add:
12432 return RISCVISD::PM2ADD;
12433 case Intrinsic::riscv_pm2add_x:
12434 return RISCVISD::PM2ADD_X;
12435 case Intrinsic::riscv_pm4addu:
12436 return RISCVISD::PM4ADDU;
12437 case Intrinsic::riscv_pm2addu:
12438 return RISCVISD::PM2ADDU;
12439 case Intrinsic::riscv_pmq2add:
12440 return RISCVISD::PMQ2ADD;
12441 case Intrinsic::riscv_pmqr2add:
12442 return RISCVISD::PMQR2ADD;
12443 case Intrinsic::riscv_pm2sadd:
12444 return RISCVISD::PM2SADD;
12445 case Intrinsic::riscv_pm2sadd_x:
12446 return RISCVISD::PM2SADD_X;
12447 case Intrinsic::riscv_pm2sub:
12448 return RISCVISD::PM2SUB;
12449 case Intrinsic::riscv_pm2sub_x:
12450 return RISCVISD::PM2SUB_X;
12451 case Intrinsic::riscv_pm4addsu:
12452 return RISCVISD::PM4ADDSU;
12453 case Intrinsic::riscv_pm2addsu:
12454 return RISCVISD::PM2ADDSU;
12455 }
12456}
12457
12458static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
12459 SDValue Rs2, const SDLoc &DL,
12460 SelectionDAG &DAG) {
12461 auto Extract = [&](SDValue V, unsigned Idx) {
12462 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx);
12463 };
12464
12465 if (Rs1.getSimpleValueType() == MVT::v4i16) {
12466 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
12467 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
12468 unsigned MulOpc, AccOpc;
12469 switch (IntNo) {
12470 default:
12471 llvm_unreachable("Unexpected RV32 horizontal multiply intrinsic");
12472 case Intrinsic::riscv_pm4add:
12473 MulOpc = RISCVISD::PM2WADD;
12474 AccOpc = RISCVISD::PM2WADDA;
12475 break;
12476 case Intrinsic::riscv_pm4addu:
12477 MulOpc = RISCVISD::PM2WADDU;
12478 AccOpc = RISCVISD::PM2WADDAU;
12479 break;
12480 case Intrinsic::riscv_pm4addsu:
12481 MulOpc = RISCVISD::PM2WADDSU;
12482 AccOpc = RISCVISD::PM2WADDASU;
12483 break;
12484 }
12485 SDValue Acc = DAG.getNode(Opcode: MulOpc, DL, VT: MVT::v2i32, N1: Rs1Lo, N2: Rs2Lo);
12486 Acc = DAG.getNode(Opcode: AccOpc, DL, VT: MVT::v2i32, N1: Acc, N2: Rs1Hi, N3: Rs2Hi);
12487 return DAG.getBitcast(VT: MVT::i64, V: Acc);
12488 }
12489
12490 assert(Rs1.getSimpleValueType() == MVT::v2i32 &&
12491 "Unexpected RV32 horizontal multiply source type");
12492 SDValue Rs1Lo = Extract(Rs1, 0);
12493 SDValue Rs1Hi = Extract(Rs1, 1);
12494 SDValue Rs2Lo = Extract(Rs2, 0);
12495 SDValue Rs2Hi = Extract(Rs2, 1);
12496
12497 if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) {
12498 unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC
12499 : RISCVISD::MQRWACC;
12500 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32);
12501 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
12502 SDValue Acc = DAG.getNode(Opcode: AccOpc, DL, VTList: VTs, Ops: {Zero, Zero, Rs1Lo, Rs2Lo});
12503 Acc = DAG.getNode(Opcode: AccOpc, DL, VTList: VTs, Ops: {Acc, Acc.getValue(R: 1), Rs1Hi, Rs2Hi});
12504 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Acc, N2: Acc.getValue(R: 1));
12505 }
12506
12507 unsigned MulOpc = ISD::SMUL_LOHI;
12508 if (IntNo == Intrinsic::riscv_pm2addu)
12509 MulOpc = ISD::UMUL_LOHI;
12510 else if (IntNo == Intrinsic::riscv_pm2addsu)
12511 MulOpc = RISCVISD::WMULSU;
12512 bool IsSub =
12513 IntNo == Intrinsic::riscv_pm2sub || IntNo == Intrinsic::riscv_pm2sub_x;
12514 if (IntNo == Intrinsic::riscv_pm2add_x || IntNo == Intrinsic::riscv_pm2sub_x)
12515 std::swap(a&: Rs2Lo, b&: Rs2Hi);
12516
12517 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32);
12518 SDValue LoMul = DAG.getNode(Opcode: MulOpc, DL, VTList: VTs, N1: Rs1Lo, N2: Rs2Lo);
12519 SDValue HiMul = DAG.getNode(Opcode: MulOpc, DL, VTList: VTs, N1: Rs1Hi, N2: Rs2Hi);
12520 unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD;
12521 SDValue Res = DAG.getNode(Opcode: Opc, DL, VTList: VTs, N1: LoMul, N2: LoMul.getValue(R: 1), N3: HiMul,
12522 N4: HiMul.getValue(R: 1));
12523 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Res, N2: Res.getValue(R: 1));
12524}
12525
12526/// Return the packed multiply-halves node for a multiply-parts intrinsic. The
12527/// scalar spelling maps to the same node; its product is the first element.
12528static unsigned getRVPMulHalvesOpcode(unsigned IntNo) {
12529 switch (IntNo) {
12530 default:
12531 llvm_unreachable("Unexpected RISC-V multiply-parts intrinsic");
12532 case Intrinsic::riscv_pmul_00:
12533 case Intrinsic::riscv_mul_00:
12534 return RISCVISD::PMUL_HALVES_00;
12535 case Intrinsic::riscv_pmul_01:
12536 case Intrinsic::riscv_mul_01:
12537 return RISCVISD::PMUL_HALVES_01;
12538 case Intrinsic::riscv_pmul_11:
12539 case Intrinsic::riscv_mul_11:
12540 return RISCVISD::PMUL_HALVES_11;
12541 case Intrinsic::riscv_pmulu_00:
12542 case Intrinsic::riscv_mulu_00:
12543 return RISCVISD::PMULU_HALVES_00;
12544 case Intrinsic::riscv_pmulu_01:
12545 case Intrinsic::riscv_mulu_01:
12546 return RISCVISD::PMULU_HALVES_01;
12547 case Intrinsic::riscv_pmulu_11:
12548 case Intrinsic::riscv_mulu_11:
12549 return RISCVISD::PMULU_HALVES_11;
12550 case Intrinsic::riscv_pmulsu_00:
12551 case Intrinsic::riscv_mulsu_00:
12552 return RISCVISD::PMULSU_HALVES_00;
12553 case Intrinsic::riscv_pmulsu_11:
12554 case Intrinsic::riscv_mulsu_11:
12555 return RISCVISD::PMULSU_HALVES_11;
12556 }
12557}
12558
12559/// Return the scalar multiply-parts intrinsic computing the first product of
12560/// packed intrinsic \p IntNo.
12561static Intrinsic::ID getRVPScalarMulPartsIntrinsic(unsigned IntNo) {
12562 switch (IntNo) {
12563 default:
12564 llvm_unreachable("Unexpected RISC-V packed multiply-parts intrinsic");
12565 case Intrinsic::riscv_pmul_00:
12566 return Intrinsic::riscv_mul_00;
12567 case Intrinsic::riscv_pmul_01:
12568 return Intrinsic::riscv_mul_01;
12569 case Intrinsic::riscv_pmul_11:
12570 return Intrinsic::riscv_mul_11;
12571 case Intrinsic::riscv_pmulu_00:
12572 return Intrinsic::riscv_mulu_00;
12573 case Intrinsic::riscv_pmulu_01:
12574 return Intrinsic::riscv_mulu_01;
12575 case Intrinsic::riscv_pmulu_11:
12576 return Intrinsic::riscv_mulu_11;
12577 case Intrinsic::riscv_pmulsu_00:
12578 return Intrinsic::riscv_mulsu_00;
12579 case Intrinsic::riscv_pmulsu_11:
12580 return Intrinsic::riscv_mulsu_11;
12581 }
12582}
12583
12584/// Return the accumulate form of multiply-parts intrinsic \p IntNo, or
12585/// Intrinsic::not_intrinsic if there is none.
12586static Intrinsic::ID getRVPMulPartsAccIntrinsic(unsigned IntNo) {
12587 switch (IntNo) {
12588 default:
12589 return Intrinsic::not_intrinsic;
12590 case Intrinsic::riscv_mul_00:
12591 return Intrinsic::riscv_macc_00;
12592 case Intrinsic::riscv_pmul_00:
12593 return Intrinsic::riscv_pmacc_00;
12594 case Intrinsic::riscv_mul_01:
12595 return Intrinsic::riscv_macc_01;
12596 case Intrinsic::riscv_pmul_01:
12597 return Intrinsic::riscv_pmacc_01;
12598 case Intrinsic::riscv_mul_11:
12599 return Intrinsic::riscv_macc_11;
12600 case Intrinsic::riscv_pmul_11:
12601 return Intrinsic::riscv_pmacc_11;
12602 case Intrinsic::riscv_mulu_00:
12603 return Intrinsic::riscv_maccu_00;
12604 case Intrinsic::riscv_pmulu_00:
12605 return Intrinsic::riscv_pmaccu_00;
12606 case Intrinsic::riscv_mulu_01:
12607 return Intrinsic::riscv_maccu_01;
12608 case Intrinsic::riscv_pmulu_01:
12609 return Intrinsic::riscv_pmaccu_01;
12610 case Intrinsic::riscv_mulu_11:
12611 return Intrinsic::riscv_maccu_11;
12612 case Intrinsic::riscv_pmulu_11:
12613 return Intrinsic::riscv_pmaccu_11;
12614 case Intrinsic::riscv_mulsu_00:
12615 return Intrinsic::riscv_maccsu_00;
12616 case Intrinsic::riscv_pmulsu_00:
12617 return Intrinsic::riscv_pmaccsu_00;
12618 case Intrinsic::riscv_mulsu_11:
12619 return Intrinsic::riscv_maccsu_11;
12620 case Intrinsic::riscv_pmulsu_11:
12621 return Intrinsic::riscv_pmaccsu_11;
12622 }
12623}
12624
12625/// Return the multiply-parts accumulate node for \p IntNo.
12626static unsigned getRVPMulAccHalvesOpcode(unsigned IntNo) {
12627 switch (IntNo) {
12628 default:
12629 llvm_unreachable("Unexpected RISC-V multiply-parts accumulate intrinsic");
12630 case Intrinsic::riscv_pmacc_00:
12631 case Intrinsic::riscv_macc_00:
12632 return RISCVISD::PMACC_HALVES_00;
12633 case Intrinsic::riscv_pmacc_01:
12634 case Intrinsic::riscv_macc_01:
12635 return RISCVISD::PMACC_HALVES_01;
12636 case Intrinsic::riscv_pmacc_11:
12637 case Intrinsic::riscv_macc_11:
12638 return RISCVISD::PMACC_HALVES_11;
12639 case Intrinsic::riscv_pmaccu_00:
12640 case Intrinsic::riscv_maccu_00:
12641 return RISCVISD::PMACCU_HALVES_00;
12642 case Intrinsic::riscv_pmaccu_01:
12643 case Intrinsic::riscv_maccu_01:
12644 return RISCVISD::PMACCU_HALVES_01;
12645 case Intrinsic::riscv_pmaccu_11:
12646 case Intrinsic::riscv_maccu_11:
12647 return RISCVISD::PMACCU_HALVES_11;
12648 case Intrinsic::riscv_pmaccsu_00:
12649 case Intrinsic::riscv_maccsu_00:
12650 return RISCVISD::PMACCSU_HALVES_00;
12651 case Intrinsic::riscv_pmaccsu_11:
12652 case Intrinsic::riscv_maccsu_11:
12653 return RISCVISD::PMACCSU_HALVES_11;
12654 }
12655}
12656
12657/// Return {opcode, rs1 lane, rs2 lane} for the word form of \p IntNo.
12658static std::tuple<unsigned, unsigned, unsigned>
12659getRVPWordMulPartsOpcodeAndLanes(unsigned IntNo) {
12660 switch (IntNo) {
12661 default:
12662 llvm_unreachable("Unexpected RISC-V multiply-parts intrinsic");
12663 case Intrinsic::riscv_mul_00:
12664 return {ISD::SMUL_LOHI, 0, 0};
12665 case Intrinsic::riscv_mul_01:
12666 return {ISD::SMUL_LOHI, 0, 1};
12667 case Intrinsic::riscv_mul_11:
12668 return {ISD::SMUL_LOHI, 1, 1};
12669 case Intrinsic::riscv_mulu_00:
12670 return {ISD::UMUL_LOHI, 0, 0};
12671 case Intrinsic::riscv_mulu_01:
12672 return {ISD::UMUL_LOHI, 0, 1};
12673 case Intrinsic::riscv_mulu_11:
12674 return {ISD::UMUL_LOHI, 1, 1};
12675 case Intrinsic::riscv_mulsu_00:
12676 return {RISCVISD::WMULSU, 0, 0};
12677 case Intrinsic::riscv_mulsu_11:
12678 return {RISCVISD::WMULSU, 1, 1};
12679 }
12680}
12681
12682/// Return {opcode, rs1 lane, rs2 lane} for the word form of accumulate
12683/// intrinsic \p IntNo.
12684static std::tuple<unsigned, unsigned, unsigned>
12685getRVPWordMulPartsAccOpcodeAndLanes(unsigned IntNo) {
12686 switch (IntNo) {
12687 default:
12688 llvm_unreachable("Unexpected RISC-V multiply-parts accumulate intrinsic");
12689 case Intrinsic::riscv_macc_00:
12690 return {RISCVISD::WMACC, 0, 0};
12691 case Intrinsic::riscv_macc_01:
12692 return {RISCVISD::WMACC, 0, 1};
12693 case Intrinsic::riscv_macc_11:
12694 return {RISCVISD::WMACC, 1, 1};
12695 case Intrinsic::riscv_maccu_00:
12696 return {RISCVISD::WMACCU, 0, 0};
12697 case Intrinsic::riscv_maccu_01:
12698 return {RISCVISD::WMACCU, 0, 1};
12699 case Intrinsic::riscv_maccu_11:
12700 return {RISCVISD::WMACCU, 1, 1};
12701 case Intrinsic::riscv_maccsu_00:
12702 return {RISCVISD::WMACCSU, 0, 0};
12703 case Intrinsic::riscv_maccsu_11:
12704 return {RISCVISD::WMACCSU, 1, 1};
12705 }
12706}
12707
12708SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
12709 SelectionDAG &DAG) const {
12710 unsigned IntNo = Op.getConstantOperandVal(i: 0);
12711 SDLoc DL(Op);
12712 MVT XLenVT = Subtarget.getXLenVT();
12713
12714 switch (IntNo) {
12715 default:
12716 break; // Don't custom lower most intrinsics.
12717 case Intrinsic::riscv_tuple_insert: {
12718 SDValue Vec = Op.getOperand(i: 1);
12719 SDValue SubVec = Op.getOperand(i: 2);
12720 SDValue Index = Op.getOperand(i: 3);
12721
12722 return DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: Op.getValueType(), N1: Vec,
12723 N2: SubVec, N3: Index);
12724 }
12725 case Intrinsic::riscv_tuple_extract: {
12726 SDValue Vec = Op.getOperand(i: 1);
12727 SDValue Index = Op.getOperand(i: 2);
12728
12729 return DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: Op.getValueType(), N1: Vec,
12730 N2: Index);
12731 }
12732 case Intrinsic::thread_pointer: {
12733 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12734 return DAG.getRegister(Reg: RISCV::X4, VT: PtrVT);
12735 }
12736 case Intrinsic::riscv_pmul_00:
12737 case Intrinsic::riscv_pmul_01:
12738 case Intrinsic::riscv_pmul_11:
12739 case Intrinsic::riscv_pmulu_00:
12740 case Intrinsic::riscv_pmulu_01:
12741 case Intrinsic::riscv_pmulu_11:
12742 case Intrinsic::riscv_pmulsu_00:
12743 case Intrinsic::riscv_pmulsu_11: {
12744 MVT VT = Op.getSimpleValueType();
12745 SDValue Rs1 = Op.getOperand(i: 1);
12746 SDValue Rs2 = Op.getOperand(i: 2);
12747 unsigned Opc = getRVPMulHalvesOpcode(IntNo);
12748 if (!Subtarget.isPExtPackedDoubleType(VT))
12749 return DAG.getNode(Opcode: Opc, DL, VT, N1: Rs1, N2: Rs2);
12750
12751 // On RV32 a 64-bit result lives in a GPR pair; compute each half with the
12752 // 32-bit form of the same product.
12753 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
12754 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
12755 if (VT == MVT::v2i32) {
12756 // Each half is a single product, described by the scalar intrinsic.
12757 SDValue Id = DAG.getTargetConstant(Val: getRVPScalarMulPartsIntrinsic(IntNo),
12758 DL, VT: MVT::i32);
12759 SDValue Lo =
12760 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32, N1: Id, N2: Rs1Lo, N3: Rs2Lo);
12761 SDValue Hi =
12762 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32, N1: Id, N2: Rs1Hi, N3: Rs2Hi);
12763 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
12764 }
12765 MVT HalfVT = VT.getHalfNumVectorElementsVT();
12766 SDValue Lo = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Rs1Lo, N2: Rs2Lo);
12767 SDValue Hi = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: Rs1Hi, N2: Rs2Hi);
12768 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
12769 }
12770 case Intrinsic::riscv_pmacc_00:
12771 case Intrinsic::riscv_pmacc_01:
12772 case Intrinsic::riscv_pmacc_11:
12773 case Intrinsic::riscv_pmaccu_00:
12774 case Intrinsic::riscv_pmaccu_01:
12775 case Intrinsic::riscv_pmaccu_11:
12776 case Intrinsic::riscv_pmaccsu_00:
12777 case Intrinsic::riscv_pmaccsu_11:
12778 case Intrinsic::riscv_macc_00:
12779 case Intrinsic::riscv_macc_01:
12780 case Intrinsic::riscv_macc_11:
12781 case Intrinsic::riscv_maccu_00:
12782 case Intrinsic::riscv_maccu_01:
12783 case Intrinsic::riscv_maccu_11:
12784 case Intrinsic::riscv_maccsu_00:
12785 case Intrinsic::riscv_maccsu_11: {
12786 MVT VT = Op.getSimpleValueType();
12787 SDValue Rd = Op.getOperand(i: 1);
12788 SDValue Rs1 = Op.getOperand(i: 2);
12789 SDValue Rs2 = Op.getOperand(i: 3);
12790 unsigned Opc = getRVPMulAccHalvesOpcode(IntNo);
12791 if (VT != MVT::v2i32 || !Subtarget.isPExtPackedDoubleType(VT))
12792 return DAG.getNode(Opcode: Opc, DL, VT, N1: Rd, N2: Rs1, N3: Rs2);
12793
12794 // On RV32 a 64-bit result lives in a GPR pair; accumulate each half with
12795 // the 32-bit form of the same product.
12796 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
12797 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
12798 SDValue Lo =
12799 DAG.getNode(Opcode: Opc, DL, VT: MVT::i32,
12800 N1: DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Rd, Idx: 0), N2: Rs1Lo, N3: Rs2Lo);
12801 SDValue Hi =
12802 DAG.getNode(Opcode: Opc, DL, VT: MVT::i32,
12803 N1: DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Rd, Idx: 1), N2: Rs1Hi, N3: Rs2Hi);
12804 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
12805 }
12806 case Intrinsic::riscv_pas:
12807 case Intrinsic::riscv_psa:
12808 case Intrinsic::riscv_psas:
12809 case Intrinsic::riscv_pssa:
12810 case Intrinsic::riscv_paas:
12811 case Intrinsic::riscv_pasa: {
12812 // v2i32 has no paired instruction on RV32; split into a pair of i32 ops
12813 // with cross-lane operands. The exchange shape is: even result uses
12814 // (S1[0], S2[1]); odd result uses (S1[1], S2[0]).
12815 if (Subtarget.is64Bit() || Op.getSimpleValueType() != MVT::v2i32)
12816 break;
12817
12818 unsigned EvenOpc, OddOpc;
12819 switch (IntNo) {
12820 case Intrinsic::riscv_pas:
12821 EvenOpc = ISD::SUB;
12822 OddOpc = ISD::ADD;
12823 break;
12824 case Intrinsic::riscv_psa:
12825 EvenOpc = ISD::ADD;
12826 OddOpc = ISD::SUB;
12827 break;
12828 case Intrinsic::riscv_psas:
12829 EvenOpc = ISD::SSUBSAT;
12830 OddOpc = ISD::SADDSAT;
12831 break;
12832 case Intrinsic::riscv_pssa:
12833 EvenOpc = ISD::SADDSAT;
12834 OddOpc = ISD::SSUBSAT;
12835 break;
12836 case Intrinsic::riscv_paas:
12837 EvenOpc = RISCVISD::ASUB;
12838 OddOpc = ISD::AVGFLOORS;
12839 break;
12840 case Intrinsic::riscv_pasa:
12841 EvenOpc = ISD::AVGFLOORS;
12842 OddOpc = RISCVISD::ASUB;
12843 break;
12844 default:
12845 llvm_unreachable("Unexpected exchanged add/sub intrinsic");
12846 }
12847
12848 SDValue S1 = Op.getOperand(i: 1);
12849 SDValue S2 = Op.getOperand(i: 2);
12850 SDValue S1Even = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S1, Idx: 0);
12851 SDValue S1Odd = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S1, Idx: 1);
12852 SDValue S2Even = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S2, Idx: 0);
12853 SDValue S2Odd = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S2, Idx: 1);
12854
12855 SDValue REven = DAG.getNode(Opcode: EvenOpc, DL, VT: MVT::i32, N1: S1Even, N2: S2Odd);
12856 SDValue ROdd = DAG.getNode(Opcode: OddOpc, DL, VT: MVT::i32, N1: S1Odd, N2: S2Even);
12857 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2i32, N1: REven, N2: ROdd);
12858 }
12859 case Intrinsic::riscv_orc_b:
12860 case Intrinsic::riscv_brev8:
12861 case Intrinsic::riscv_sha256sig0:
12862 case Intrinsic::riscv_sha256sig1:
12863 case Intrinsic::riscv_sha256sum0:
12864 case Intrinsic::riscv_sha256sum1:
12865 case Intrinsic::riscv_sm3p0:
12866 case Intrinsic::riscv_sm3p1: {
12867 unsigned Opc;
12868 switch (IntNo) {
12869 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
12870 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
12871 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
12872 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
12873 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
12874 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
12875 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
12876 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
12877 }
12878
12879 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
12880 }
12881 case Intrinsic::riscv_sm4ks:
12882 case Intrinsic::riscv_sm4ed: {
12883 unsigned Opc =
12884 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
12885
12886 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
12887 N3: Op.getOperand(i: 3));
12888 }
12889 case Intrinsic::riscv_zip:
12890 case Intrinsic::riscv_unzip: {
12891 unsigned Opc =
12892 IntNo == Intrinsic::riscv_zip ? RISCVISD::ZIP : RISCVISD::UNZIP;
12893 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
12894 }
12895 case Intrinsic::riscv_mopr:
12896 return DAG.getNode(Opcode: RISCVISD::MOP_R, DL, VT: XLenVT, N1: Op.getOperand(i: 1),
12897 N2: Op.getOperand(i: 2));
12898
12899 case Intrinsic::riscv_moprr: {
12900 return DAG.getNode(Opcode: RISCVISD::MOP_RR, DL, VT: XLenVT, N1: Op.getOperand(i: 1),
12901 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
12902 }
12903 case Intrinsic::riscv_clmulh:
12904 case Intrinsic::riscv_clmulr: {
12905 unsigned Opc = IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
12906 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
12907 }
12908 case Intrinsic::riscv_paadd:
12909 case Intrinsic::riscv_paaddu:
12910 case Intrinsic::riscv_pasub:
12911 case Intrinsic::riscv_pasubu:
12912 case Intrinsic::riscv_pabd:
12913 case Intrinsic::riscv_pabdu:
12914 case Intrinsic::riscv_psabs: {
12915 unsigned Opc;
12916 switch (IntNo) {
12917 case Intrinsic::riscv_paadd:
12918 Opc = ISD::AVGFLOORS;
12919 break;
12920 case Intrinsic::riscv_paaddu:
12921 Opc = ISD::AVGFLOORU;
12922 break;
12923 case Intrinsic::riscv_pasub:
12924 Opc = RISCVISD::ASUB;
12925 break;
12926 case Intrinsic::riscv_pasubu:
12927 Opc = RISCVISD::ASUBU;
12928 break;
12929 case Intrinsic::riscv_pabd:
12930 Opc = ISD::ABDS;
12931 break;
12932 case Intrinsic::riscv_pabdu:
12933 Opc = ISD::ABDU;
12934 break;
12935 case Intrinsic::riscv_psabs:
12936 Opc = RISCVISD::PSABS;
12937 break;
12938 }
12939
12940 if (IntNo == Intrinsic::riscv_psabs)
12941 return DAG.getNode(Opcode: Opc, DL, VT: Op.getValueType(), Operand: Op.getOperand(i: 1));
12942
12943 return DAG.getNode(Opcode: Opc, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 1),
12944 N2: Op.getOperand(i: 2));
12945 }
12946 case Intrinsic::riscv_pnclipp:
12947 case Intrinsic::riscv_pnclipup: {
12948 bool IsSigned = IntNo == Intrinsic::riscv_pnclipp;
12949 EVT VT = Op.getValueType();
12950 SDValue Rs1 = Op.getOperand(i: 1);
12951 SDValue Rs2 = Op.getOperand(i: 2);
12952 if (Subtarget.is64Bit()) {
12953 if (VT == MVT::v2i32 && !Rs1.getValueType().isVector()) {
12954 unsigned WOpc = IsSigned ? RISCVISD::PNCLIPP_W : RISCVISD::PNCLIPUP_W;
12955 return DAG.getNode(Opcode: WOpc, DL, VT, N1: Rs1, N2: Rs2);
12956 }
12957 unsigned Opc = IsSigned ? RISCVISD::PNCLIPP : RISCVISD::PNCLIPUP;
12958 return DAG.getNode(Opcode: Opc, DL, VT, N1: Rs1, N2: Rs2);
12959 }
12960
12961 MVT XLenVT = Subtarget.getXLenVT();
12962 SDValue Shift = DAG.getTargetConstant(Val: 0, DL, VT: XLenVT);
12963 if (VT == MVT::v4i8) {
12964 unsigned ClipOpc = IsSigned ? RISCVISD::PNCLIP : RISCVISD::PNCLIPU;
12965 SDValue Pair = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16, N1: Rs1, N2: Rs2);
12966 return DAG.getNode(Opcode: ClipOpc, DL, VT, N1: Pair, N2: Shift);
12967 }
12968 if (VT == MVT::v2i16) {
12969 unsigned ClipOpc = IsSigned ? RISCVISD::PNCLIP : RISCVISD::PNCLIPU;
12970 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2i32, N1: Rs1, N2: Rs2);
12971 return DAG.getNode(Opcode: ClipOpc, DL, VT, N1: Pair, N2: Shift);
12972 }
12973 if (VT == MVT::v2i32) {
12974 unsigned ClipOpc = IsSigned ? RISCVISD::NCLIP : RISCVISD::NCLIPU;
12975 auto [Rs1Lo, Rs1Hi] = DAG.SplitScalar(N: Rs1, DL, LoVT: XLenVT, HiVT: XLenVT);
12976 auto [Rs2Lo, Rs2Hi] = DAG.SplitScalar(N: Rs2, DL, LoVT: XLenVT, HiVT: XLenVT);
12977 SDValue Lo = DAG.getNode(Opcode: ClipOpc, DL, VT: XLenVT, N1: Rs1Lo, N2: Rs1Hi, N3: Shift);
12978 SDValue Hi = DAG.getNode(Opcode: ClipOpc, DL, VT: XLenVT, N1: Rs2Lo, N2: Rs2Hi, N3: Shift);
12979 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
12980 }
12981 if (VT == MVT::v8i8 || VT == MVT::v4i16) {
12982 unsigned ClipOpc = IsSigned ? RISCVISD::PNCLIP : RISCVISD::PNCLIPU;
12983 MVT HalfVT = VT == MVT::v8i8 ? MVT::v4i8 : MVT::v2i16;
12984 SDValue Lo = DAG.getNode(Opcode: ClipOpc, DL, VT: HalfVT, N1: Rs1, N2: Shift);
12985 SDValue Hi = DAG.getNode(Opcode: ClipOpc, DL, VT: HalfVT, N1: Rs2, N2: Shift);
12986 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
12987 }
12988
12989 llvm_unreachable("unexpected VT for pnclipp/pnclipup on RV32");
12990 }
12991 case Intrinsic::riscv_mqacc_00:
12992 case Intrinsic::riscv_mqacc_01:
12993 case Intrinsic::riscv_mqacc_11:
12994 case Intrinsic::riscv_mqracc_00:
12995 case Intrinsic::riscv_mqracc_01:
12996 case Intrinsic::riscv_mqracc_11:
12997 case Intrinsic::riscv_pmqacc_h00:
12998 case Intrinsic::riscv_pmqacc_h01:
12999 case Intrinsic::riscv_pmqacc_h11:
13000 case Intrinsic::riscv_pmqracc_h00:
13001 case Intrinsic::riscv_pmqracc_h01:
13002 case Intrinsic::riscv_pmqracc_h11: {
13003 EVT VT = Op.getValueType();
13004 SDValue Rd = Op.getOperand(i: 1);
13005 SDValue Rs1 = Op.getOperand(i: 2);
13006 SDValue Rs2 = Op.getOperand(i: 3);
13007 MVT XLenVT = Subtarget.getXLenVT();
13008
13009 bool IsScalarHalfword = VT == MVT::i32;
13010 if (Subtarget.is64Bit() && IsScalarHalfword)
13011 return SDValue();
13012
13013 if (VT == MVT::v2i32 && Rs1.getSimpleValueType() == MVT::v4i16) {
13014 if (Subtarget.is64Bit()) {
13015 unsigned Opc = getRVPQFormatAccOpcode(IntNo);
13016 return DAG.getNode(Opcode: Opc, DL, VT, N1: Rd, N2: Rs1, N3: Rs2);
13017 }
13018
13019 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
13020 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
13021 SDValue RdLo = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rd, Idx: 0);
13022 SDValue RdHi = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rd, Idx: 1);
13023 unsigned ScalarOpc = getRVPQFormatAccScalarOpcode(IntNo);
13024 SDValue Lo = DAG.getNode(Opcode: ScalarOpc, DL, VT: XLenVT, N1: RdLo, N2: Rs1Lo, N3: Rs2Lo);
13025 SDValue Hi = DAG.getNode(Opcode: ScalarOpc, DL, VT: XLenVT, N1: RdHi, N2: Rs1Hi, N3: Rs2Hi);
13026 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
13027 }
13028
13029 if (VT == MVT::i32 && !Subtarget.is64Bit()) {
13030 unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo);
13031 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Rd, N2: Rs1, N3: Rs2);
13032 }
13033
13034 if (VT == MVT::i64 && Subtarget.is64Bit()) {
13035 unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo);
13036 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Rd, N2: Rs1, N3: Rs2);
13037 }
13038
13039 return SDValue();
13040 }
13041 case Intrinsic::riscv_pmulq:
13042 case Intrinsic::riscv_pmulqr: {
13043 unsigned Opc;
13044 switch (IntNo) {
13045 case Intrinsic::riscv_pmulq:
13046 Opc = RISCVISD::MULQ;
13047 break;
13048 case Intrinsic::riscv_pmulqr:
13049 Opc = RISCVISD::MULQR;
13050 break;
13051 }
13052
13053 EVT VT = Op.getValueType();
13054 SDValue Rs1 = Op.getOperand(i: 1);
13055 SDValue Rs2 = Op.getOperand(i: 2);
13056
13057 // On RV32 the 64-bit packed vectors (v2i32, v4i16) have no single Q-format
13058 // multiply instruction. Split v4i16 into two v2i16 halves (each lowered to
13059 // a single pmulq.h/pmulqr.h via the existing patterns), and v2i32 into two
13060 // scalar mulq/mulqr (matched by the RV32 PatGprGpr patterns).
13061 if (!Subtarget.is64Bit()) {
13062 if (VT == MVT::v2i32) {
13063 MVT XLenVT = Subtarget.getXLenVT();
13064 SDValue Lo1 = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rs1, Idx: 0);
13065 SDValue Lo2 = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rs2, Idx: 0);
13066 SDValue Hi1 = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rs1, Idx: 1);
13067 SDValue Hi2 = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rs2, Idx: 1);
13068 SDValue LoRes = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Lo1, N2: Lo2);
13069 SDValue HiRes = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Hi1, N2: Hi2);
13070 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: LoRes, N2: HiRes);
13071 }
13072 if (VT == MVT::v4i16) {
13073 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
13074 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
13075 SDValue LoRes = DAG.getNode(Opcode: Opc, DL, VT: MVT::v2i16, N1: Rs1Lo, N2: Rs2Lo);
13076 SDValue HiRes = DAG.getNode(Opcode: Opc, DL, VT: MVT::v2i16, N1: Rs1Hi, N2: Rs2Hi);
13077 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: LoRes, N2: HiRes);
13078 }
13079 }
13080
13081 return DAG.getNode(Opcode: Opc, DL, VT, N1: Rs1, N2: Rs2);
13082 }
13083 case Intrinsic::riscv_pssha:
13084 case Intrinsic::riscv_psshar:
13085 case Intrinsic::riscv_psshl:
13086 case Intrinsic::riscv_psshlr: {
13087 SDValue ShAmt = Op.getOperand(i: 2);
13088 ShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: XLenVT);
13089 return DAG.getNode(Opcode: getRVPShiftOpcode(IntNo), DL, VT: Op.getValueType(),
13090 N1: Op.getOperand(i: 1), N2: ShAmt);
13091 }
13092 case Intrinsic::riscv_psext_b:
13093 case Intrinsic::riscv_psext_h: {
13094 EVT VT = Op.getValueType();
13095 if (!VT.isSimple() || !Subtarget.isPExtPackedType(VT: VT.getSimpleVT()))
13096 reportFatalUsageError(reason: "unsupported llvm.riscv.psext intrinsic");
13097
13098 MVT SimpleVT = VT.getSimpleVT();
13099 unsigned SrcEltBits = IntNo == Intrinsic::riscv_psext_b ? 8 : 16;
13100 if (SrcEltBits >= SimpleVT.getScalarSizeInBits())
13101 reportFatalUsageError(reason: "unsupported llvm.riscv.psext intrinsic");
13102
13103 MVT ExtVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltBits),
13104 EC: SimpleVT.getVectorElementCount());
13105 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: SimpleVT, N1: Op.getOperand(i: 1),
13106 N2: DAG.getValueType(ExtVT));
13107 }
13108 case Intrinsic::riscv_pzext_b:
13109 case Intrinsic::riscv_pzext_h: {
13110 EVT VT = Op.getValueType();
13111 if (!VT.isSimple() || !Subtarget.isPExtPackedType(VT: VT.getSimpleVT()))
13112 reportFatalUsageError(reason: "unsupported llvm.riscv.pzext intrinsic");
13113
13114 MVT SimpleVT = VT.getSimpleVT();
13115 unsigned SrcEltBits = IntNo == Intrinsic::riscv_pzext_b ? 8 : 16;
13116 if (SimpleVT.getScalarSizeInBits() != SrcEltBits * 2)
13117 reportFatalUsageError(reason: "unsupported llvm.riscv.pzext intrinsic");
13118
13119 return lowerPZExt(Src: Op.getOperand(i: 1), DL, DAG, Subtarget);
13120 }
13121 case Intrinsic::riscv_pabdsumu:
13122 case Intrinsic::riscv_pabdsumau: {
13123 // On RV32 an i32-result absolute difference sum over a 64-bit (GPRPair)
13124 // source has no paired instruction. Split into two v4i8 halves: reduce the
13125 // low half (folding in rd when accumulating), then accumulate the high half
13126 // into that partial sum.
13127 SDValue Rs1 = Op.getOperand(i: Op.getNumOperands() - 2);
13128 SDValue Rs2 = Op.getOperand(i: Op.getNumOperands() - 1);
13129 if (Subtarget.is64Bit() || Rs1.getSimpleValueType() != MVT::v8i8)
13130 return SDValue();
13131 bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
13132 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
13133 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
13134 SDValue AbdsumuId =
13135 DAG.getTargetConstant(Val: Intrinsic::riscv_pabdsumu, DL, VT: MVT::i32);
13136 SDValue AbdsumauId =
13137 DAG.getTargetConstant(Val: Intrinsic::riscv_pabdsumau, DL, VT: MVT::i32);
13138 SDValue Lo = IsAcc ? DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
13139 N1: AbdsumauId, N2: Op.getOperand(i: 1), N3: Rs1Lo, N4: Rs2Lo)
13140 : DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
13141 N1: AbdsumuId, N2: Rs1Lo, N3: Rs2Lo);
13142 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32, N1: AbdsumauId, N2: Lo,
13143 N3: Rs1Hi, N4: Rs2Hi);
13144 }
13145 case Intrinsic::riscv_pmulh:
13146 case Intrinsic::riscv_pmulhr:
13147 case Intrinsic::riscv_pmulhu:
13148 case Intrinsic::riscv_pmulhru:
13149 case Intrinsic::riscv_pmulhsu:
13150 case Intrinsic::riscv_pmulhrsu: {
13151 EVT VT = Op.getValueType();
13152 unsigned Opc = getRVPMulHighOpcode(IntNo);
13153
13154 // RV32 has no single instruction for 64-bit packed multiply high. Split
13155 // v4i16 into two v2i16 packed operations, and split v2i32 into scalar i32
13156 // operations so isel can use MULH*.
13157 if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
13158 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Op.getOperand(i: 1), DL);
13159 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Op.getOperand(i: 2), DL);
13160 SDValue Lo = DAG.getNode(Opcode: Opc, DL, VT: MVT::v2i16, N1: Rs1Lo, N2: Rs2Lo);
13161 SDValue Hi = DAG.getNode(Opcode: Opc, DL, VT: MVT::v2i16, N1: Rs1Hi, N2: Rs2Hi);
13162 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
13163 }
13164
13165 if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
13166 auto Extract = [&](SDValue V, unsigned Idx) {
13167 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx);
13168 };
13169 SDValue Rs1 = Op.getOperand(i: 1);
13170 SDValue Rs2 = Op.getOperand(i: 2);
13171 SDValue Lo =
13172 DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, N1: Extract(Rs1, 0), N2: Extract(Rs2, 0));
13173 SDValue Hi =
13174 DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, N1: Extract(Rs1, 1), N2: Extract(Rs2, 1));
13175 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
13176 }
13177
13178 return DAG.getNode(Opcode: Opc, DL, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
13179 }
13180 case Intrinsic::riscv_pmhacc:
13181 case Intrinsic::riscv_pmhracc:
13182 case Intrinsic::riscv_pmhaccu:
13183 case Intrinsic::riscv_pmhraccu:
13184 case Intrinsic::riscv_pmhaccsu:
13185 case Intrinsic::riscv_pmhraccsu: {
13186 EVT VT = Op.getValueType();
13187 unsigned MulOpc = getRVPMulHighAccumulateOpcode(IntNo);
13188 SDValue Rd = Op.getOperand(i: 1);
13189 SDValue Rs1 = Op.getOperand(i: 2);
13190 SDValue Rs2 = Op.getOperand(i: 3);
13191
13192 // RV32 has no single instruction for 64-bit packed multiply high
13193 // accumulate. Split v4i16 into two v2i16 packed operations, and split
13194 // v2i32 into scalar i32 operations.
13195 if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
13196 auto [RdLo, RdHi] = DAG.SplitVector(N: Rd, DL);
13197 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
13198 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
13199 SDValue Lo = DAG.getNode(Opcode: MulOpc, DL, VT: MVT::v2i16, N1: RdLo, N2: Rs1Lo, N3: Rs2Lo);
13200 SDValue Hi = DAG.getNode(Opcode: MulOpc, DL, VT: MVT::v2i16, N1: RdHi, N2: Rs1Hi, N3: Rs2Hi);
13201 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
13202 }
13203
13204 if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
13205 auto Extract = [&](SDValue V, unsigned Idx) {
13206 return DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx);
13207 };
13208 SDValue Lo = DAG.getNode(Opcode: MulOpc, DL, VT: MVT::i32, N1: Extract(Rd, 0),
13209 N2: Extract(Rs1, 0), N3: Extract(Rs2, 0));
13210 SDValue Hi = DAG.getNode(Opcode: MulOpc, DL, VT: MVT::i32, N1: Extract(Rd, 1),
13211 N2: Extract(Rs1, 1), N3: Extract(Rs2, 1));
13212 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
13213 }
13214
13215 return DAG.getNode(Opcode: MulOpc, DL, VT, N1: Rd, N2: Rs1, N3: Rs2);
13216 }
13217 case Intrinsic::riscv_pm4add:
13218 case Intrinsic::riscv_pm2add:
13219 case Intrinsic::riscv_pm2add_x:
13220 case Intrinsic::riscv_pm4addu:
13221 case Intrinsic::riscv_pm2addu:
13222 case Intrinsic::riscv_pmq2add:
13223 case Intrinsic::riscv_pmqr2add:
13224 case Intrinsic::riscv_pm2sadd:
13225 case Intrinsic::riscv_pm2sadd_x:
13226 case Intrinsic::riscv_pm2sub:
13227 case Intrinsic::riscv_pm2sub_x:
13228 case Intrinsic::riscv_pm4addsu:
13229 case Intrinsic::riscv_pm2addsu: {
13230 EVT VT = Op.getValueType();
13231 unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
13232 SDValue Rs1 = Op.getOperand(i: 1);
13233 SDValue Rs2 = Op.getOperand(i: 2);
13234
13235 // RV32 applies the 32-bit instruction independently to both halves of a
13236 // 64-bit packed input.
13237 if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
13238 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
13239 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
13240 SDValue Lo = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, N1: Rs1Lo, N2: Rs2Lo);
13241 SDValue Hi = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, N1: Rs1Hi, N2: Rs2Hi);
13242 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
13243 }
13244
13245 return DAG.getNode(Opcode: Opc, DL, VT, N1: Rs1, N2: Rs2);
13246 }
13247 case Intrinsic::riscv_pmerge: {
13248 EVT VT = Op.getValueType();
13249 auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
13250 EVT ResultVT) {
13251 MVT IntVT = MVT::getIntegerVT(BitWidth: ResultVT.getSizeInBits());
13252 SDValue Res =
13253 DAG.getNode(Opcode: RISCVISD::MERGE, DL, VT: IntVT, N1: DAG.getBitcast(VT: IntVT, V: Mask),
13254 N2: DAG.getBitcast(VT: IntVT, V: Rs1), N3: DAG.getBitcast(VT: IntVT, V: Rs2));
13255 return DAG.getBitcast(VT: ResultVT, V: Res);
13256 };
13257
13258 // 64-bit packed types on RV32: split into two 32-bit halves. v2i32 has no
13259 // legal 32-bit vector half, so bitcast it to v4i16 (same 64 bits) first;
13260 // the merge result is identical.
13261 if (!Subtarget.is64Bit() &&
13262 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32)) {
13263 EVT WorkVT = VT == MVT::v2i32 ? EVT(MVT::v4i16) : VT;
13264 SDValue Rs1 = DAG.getBitcast(VT: WorkVT, V: Op.getOperand(i: 1));
13265 SDValue Rs2 = DAG.getBitcast(VT: WorkVT, V: Op.getOperand(i: 2));
13266 SDValue Mask = DAG.getBitcast(VT: WorkVT, V: Op.getOperand(i: 3));
13267 MVT HalfVT = WorkVT == MVT::v8i8 ? MVT::v4i8 : MVT::v2i16;
13268 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL, LoVT: HalfVT, HiVT: HalfVT);
13269 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL, LoVT: HalfVT, HiVT: HalfVT);
13270 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Mask, DL, LoVT: HalfVT, HiVT: HalfVT);
13271 SDValue ResLo = buildMerge(Rs1Lo, Rs2Lo, MaskLo, HalfVT);
13272 SDValue ResHi = buildMerge(Rs1Hi, Rs2Hi, MaskHi, HalfVT);
13273 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WorkVT, N1: ResLo, N2: ResHi);
13274 return DAG.getBitcast(VT, V: Res);
13275 }
13276
13277 return buildMerge(Op.getOperand(i: 1), Op.getOperand(i: 2), Op.getOperand(i: 3), VT);
13278 }
13279 case Intrinsic::experimental_get_vector_length:
13280 return lowerGetVectorLength(N: Op.getNode(), DAG, Subtarget);
13281 case Intrinsic::riscv_vmv_x_s: {
13282 SDValue Res = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
13283 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: Res);
13284 }
13285 case Intrinsic::riscv_vfmv_f_s:
13286 return DAG.getExtractVectorElt(DL, VT: Op.getValueType(), Vec: Op.getOperand(i: 1), Idx: 0);
13287 case Intrinsic::riscv_vmv_v_x:
13288 return lowerScalarSplat(Passthru: Op.getOperand(i: 1), Scalar: Op.getOperand(i: 2),
13289 VL: Op.getOperand(i: 3), VT: Op.getSimpleValueType(), DL, DAG,
13290 Subtarget);
13291 case Intrinsic::riscv_vfmv_v_f:
13292 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: Op.getValueType(),
13293 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
13294 case Intrinsic::riscv_vmv_s_x: {
13295 SDValue Scalar = Op.getOperand(i: 2);
13296
13297 if (Scalar.getValueType().bitsLE(VT: XLenVT)) {
13298 Scalar = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Scalar);
13299 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT: Op.getValueType(),
13300 N1: Op.getOperand(i: 1), N2: Scalar, N3: Op.getOperand(i: 3));
13301 }
13302
13303 assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
13304
13305 // This is an i64 value that lives in two scalar registers. We have to
13306 // insert this in a convoluted way. First we build vXi64 splat containing
13307 // the two values that we assemble using some bit math. Next we'll use
13308 // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
13309 // to merge element 0 from our splat into the source vector.
13310 // FIXME: This is probably not the best way to do this, but it is
13311 // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
13312 // point.
13313 // sw lo, (a0)
13314 // sw hi, 4(a0)
13315 // vlse vX, (a0)
13316 //
13317 // vid.v vVid
13318 // vmseq.vx mMask, vVid, 0
13319 // vmerge.vvm vDest, vSrc, vVal, mMask
13320 MVT VT = Op.getSimpleValueType();
13321 SDValue Vec = Op.getOperand(i: 1);
13322 SDValue VL = getVLOperand(Op);
13323
13324 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Passthru: SDValue(), Scalar, VL, DAG);
13325 if (Op.getOperand(i: 1).isUndef())
13326 return SplattedVal;
13327 SDValue SplattedIdx =
13328 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
13329 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N3: VL);
13330
13331 MVT MaskVT = getMaskTypeFor(VecVT: VT);
13332 SDValue Mask = getAllOnesMask(VecVT: VT, VL, DL, DAG);
13333 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT, N1: Mask, N2: VL);
13334 SDValue SelectCond =
13335 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: MaskVT,
13336 Ops: {VID, SplattedIdx, DAG.getCondCode(Cond: ISD::SETEQ),
13337 DAG.getUNDEF(VT: MaskVT), Mask, VL});
13338 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: SelectCond, N2: SplattedVal,
13339 N3: Vec, N4: DAG.getUNDEF(VT), N5: VL);
13340 }
13341 case Intrinsic::riscv_vfmv_s_f:
13342 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT: Op.getValueType(),
13343 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
13344 // EGS * EEW >= 128 bits
13345 case Intrinsic::riscv_vaesdf_vv:
13346 case Intrinsic::riscv_vaesdf_vs:
13347 case Intrinsic::riscv_vaesdm_vv:
13348 case Intrinsic::riscv_vaesdm_vs:
13349 case Intrinsic::riscv_vaesef_vv:
13350 case Intrinsic::riscv_vaesef_vs:
13351 case Intrinsic::riscv_vaesem_vv:
13352 case Intrinsic::riscv_vaesem_vs:
13353 case Intrinsic::riscv_vaeskf1:
13354 case Intrinsic::riscv_vaeskf2:
13355 case Intrinsic::riscv_vaesz_vs:
13356 case Intrinsic::riscv_vsm4k:
13357 case Intrinsic::riscv_vsm4r_vv:
13358 case Intrinsic::riscv_vsm4r_vs: {
13359 if (!isValidEGW(EGS: 4, VT: Op.getSimpleValueType(), Subtarget) ||
13360 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget) ||
13361 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 2).getSimpleValueType(), Subtarget))
13362 reportFatalUsageError(reason: "EGW should be greater than or equal to 4 * SEW.");
13363 return Op;
13364 }
13365 // EGS * EEW >= 256 bits
13366 case Intrinsic::riscv_vsm3c:
13367 case Intrinsic::riscv_vsm3me: {
13368 if (!isValidEGW(EGS: 8, VT: Op.getSimpleValueType(), Subtarget) ||
13369 !isValidEGW(EGS: 8, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget))
13370 reportFatalUsageError(reason: "EGW should be greater than or equal to 8 * SEW.");
13371 return Op;
13372 }
13373 // zvknha(SEW=32)/zvknhb(SEW=[32|64])
13374 case Intrinsic::riscv_vsha2ch:
13375 case Intrinsic::riscv_vsha2cl:
13376 case Intrinsic::riscv_vsha2ms: {
13377 if (Op->getSimpleValueType(ResNo: 0).getScalarSizeInBits() == 64 &&
13378 !Subtarget.hasStdExtZvknhb())
13379 reportFatalUsageError(reason: "SEW=64 needs Zvknhb to be enabled.");
13380 if (!isValidEGW(EGS: 4, VT: Op.getSimpleValueType(), Subtarget) ||
13381 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget) ||
13382 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 2).getSimpleValueType(), Subtarget))
13383 reportFatalUsageError(reason: "EGW should be greater than or equal to 4 * SEW.");
13384 return Op;
13385 }
13386 case Intrinsic::riscv_sf_vc_v_x:
13387 case Intrinsic::riscv_sf_vc_v_i:
13388 case Intrinsic::riscv_sf_vc_v_xv:
13389 case Intrinsic::riscv_sf_vc_v_iv:
13390 case Intrinsic::riscv_sf_vc_v_vv:
13391 case Intrinsic::riscv_sf_vc_v_fv:
13392 case Intrinsic::riscv_sf_vc_v_xvv:
13393 case Intrinsic::riscv_sf_vc_v_ivv:
13394 case Intrinsic::riscv_sf_vc_v_vvv:
13395 case Intrinsic::riscv_sf_vc_v_fvv:
13396 case Intrinsic::riscv_sf_vc_v_xvw:
13397 case Intrinsic::riscv_sf_vc_v_ivw:
13398 case Intrinsic::riscv_sf_vc_v_vvw:
13399 case Intrinsic::riscv_sf_vc_v_fvw: {
13400 MVT VT = Op.getSimpleValueType();
13401
13402 SmallVector<SDValue> Operands{Op->op_values()};
13403 processVCIXOperands(OrigOp: Op, Operands, DAG);
13404
13405 MVT RetVT = VT;
13406 if (VT.isFixedLengthVector())
13407 RetVT = getContainerForFixedLengthVector(VT);
13408 else if (VT.isFloatingPoint())
13409 RetVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits()),
13410 EC: VT.getVectorElementCount());
13411
13412 SDValue NewNode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: RetVT, Ops: Operands);
13413
13414 if (VT.isFixedLengthVector())
13415 NewNode = convertFromScalableVector(VT, V: NewNode, DAG, Subtarget);
13416 else if (VT.isFloatingPoint())
13417 NewNode = DAG.getBitcast(VT, V: NewNode);
13418
13419 if (Op == NewNode)
13420 break;
13421
13422 return NewNode;
13423 }
13424 }
13425
13426 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
13427}
13428
13429static inline SDValue getVCIXISDNodeWCHAIN(SDValue Op, SelectionDAG &DAG,
13430 unsigned Type) {
13431 SDLoc DL(Op);
13432 SmallVector<SDValue> Operands{Op->op_values()};
13433 Operands.erase(CI: Operands.begin() + 1);
13434
13435 const RISCVSubtarget &Subtarget =
13436 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
13437 MVT VT = Op.getSimpleValueType();
13438 MVT RetVT = VT;
13439 MVT FloatVT = VT;
13440
13441 if (VT.isFloatingPoint()) {
13442 RetVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits()),
13443 EC: VT.getVectorElementCount());
13444 FloatVT = RetVT;
13445 }
13446 if (VT.isFixedLengthVector())
13447 RetVT = getContainerForFixedLengthVector(VT: RetVT, Subtarget);
13448
13449 processVCIXOperands(OrigOp: Op, Operands, DAG);
13450
13451 SDVTList VTs = DAG.getVTList(VTs: {RetVT, MVT::Other});
13452 SDValue NewNode = DAG.getNode(Opcode: Type, DL, VTList: VTs, Ops: Operands);
13453 SDValue Chain = NewNode.getValue(R: 1);
13454
13455 if (VT.isFixedLengthVector())
13456 NewNode = convertFromScalableVector(VT: FloatVT, V: NewNode, DAG, Subtarget);
13457 if (VT.isFloatingPoint())
13458 NewNode = DAG.getBitcast(VT, V: NewNode);
13459
13460 NewNode = DAG.getMergeValues(Ops: {NewNode, Chain}, dl: DL);
13461
13462 return NewNode;
13463}
13464
13465static inline SDValue getVCIXISDNodeVOID(SDValue Op, SelectionDAG &DAG,
13466 unsigned Type) {
13467 SmallVector<SDValue> Operands{Op->op_values()};
13468 Operands.erase(CI: Operands.begin() + 1);
13469 processVCIXOperands(OrigOp: Op, Operands, DAG);
13470
13471 return DAG.getNode(Opcode: Type, DL: SDLoc(Op), VT: Op.getValueType(), Ops: Operands);
13472}
13473
13474static SDValue
13475lowerFixedVectorSegLoadIntrinsics(unsigned IntNo, SDValue Op,
13476 const RISCVSubtarget &Subtarget,
13477 SelectionDAG &DAG) {
13478 bool IsStrided;
13479 switch (IntNo) {
13480 case Intrinsic::riscv_seg2_load_mask:
13481 case Intrinsic::riscv_seg3_load_mask:
13482 case Intrinsic::riscv_seg4_load_mask:
13483 case Intrinsic::riscv_seg5_load_mask:
13484 case Intrinsic::riscv_seg6_load_mask:
13485 case Intrinsic::riscv_seg7_load_mask:
13486 case Intrinsic::riscv_seg8_load_mask:
13487 IsStrided = false;
13488 break;
13489 case Intrinsic::riscv_sseg2_load_mask:
13490 case Intrinsic::riscv_sseg3_load_mask:
13491 case Intrinsic::riscv_sseg4_load_mask:
13492 case Intrinsic::riscv_sseg5_load_mask:
13493 case Intrinsic::riscv_sseg6_load_mask:
13494 case Intrinsic::riscv_sseg7_load_mask:
13495 case Intrinsic::riscv_sseg8_load_mask:
13496 IsStrided = true;
13497 break;
13498 default:
13499 llvm_unreachable("unexpected intrinsic ID");
13500 };
13501
13502 static const Intrinsic::ID VlsegInts[7] = {
13503 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
13504 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
13505 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
13506 Intrinsic::riscv_vlseg8_mask};
13507 static const Intrinsic::ID VlssegInts[7] = {
13508 Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
13509 Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
13510 Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
13511 Intrinsic::riscv_vlsseg8_mask};
13512
13513 SDLoc DL(Op);
13514 unsigned NF = Op->getNumValues() - 1;
13515 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
13516 MVT XLenVT = Subtarget.getXLenVT();
13517 MVT VT = Op->getSimpleValueType(ResNo: 0);
13518 MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
13519 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
13520 ContainerVT.getScalarSizeInBits();
13521 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: NF);
13522
13523 // Operands: (chain, int_id, pointer, mask, vl) or
13524 // (chain, int_id, pointer, offset, mask, vl)
13525 SDValue VL = Op.getOperand(i: Op.getNumOperands() - 1);
13526 SDValue Mask = Op.getOperand(i: Op.getNumOperands() - 2);
13527 MVT MaskVT = Mask.getSimpleValueType();
13528 MVT MaskContainerVT = ::getContainerForFixedLengthVector(VT: MaskVT, Subtarget);
13529 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
13530
13531 SDValue IntID = DAG.getTargetConstant(
13532 Val: IsStrided ? VlssegInts[NF - 2] : VlsegInts[NF - 2], DL, VT: XLenVT);
13533 auto *Load = cast<MemIntrinsicSDNode>(Val&: Op);
13534
13535 SDVTList VTs = DAG.getVTList(VTs: {VecTupTy, MVT::Other});
13536 SmallVector<SDValue, 9> Ops = {
13537 Load->getChain(),
13538 IntID,
13539 DAG.getUNDEF(VT: VecTupTy),
13540 Op.getOperand(i: 2),
13541 Mask,
13542 VL,
13543 DAG.getTargetConstant(
13544 Val: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, VT: XLenVT),
13545 DAG.getTargetConstant(Val: Log2_64(Value: VT.getScalarSizeInBits()), DL, VT: XLenVT)};
13546 // Insert the stride operand.
13547 if (IsStrided)
13548 Ops.insert(I: std::next(x: Ops.begin(), n: 4), Elt: Op.getOperand(i: 3));
13549
13550 SDValue Result =
13551 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
13552 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
13553 SmallVector<SDValue, 9> Results;
13554 for (unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
13555 SDValue SubVec = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: ContainerVT,
13556 N1: Result.getValue(R: 0),
13557 N2: DAG.getTargetConstant(Val: RetIdx, DL, VT: MVT::i32));
13558 Results.push_back(Elt: convertFromScalableVector(VT, V: SubVec, DAG, Subtarget));
13559 }
13560 Results.push_back(Elt: Result.getValue(R: 1));
13561 return DAG.getMergeValues(Ops: Results, dl: DL);
13562}
13563
13564SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
13565 SelectionDAG &DAG) const {
13566 unsigned IntNo = Op.getConstantOperandVal(i: 1);
13567 switch (IntNo) {
13568 default:
13569 break;
13570 case Intrinsic::riscv_seg2_load_mask:
13571 case Intrinsic::riscv_seg3_load_mask:
13572 case Intrinsic::riscv_seg4_load_mask:
13573 case Intrinsic::riscv_seg5_load_mask:
13574 case Intrinsic::riscv_seg6_load_mask:
13575 case Intrinsic::riscv_seg7_load_mask:
13576 case Intrinsic::riscv_seg8_load_mask:
13577 case Intrinsic::riscv_sseg2_load_mask:
13578 case Intrinsic::riscv_sseg3_load_mask:
13579 case Intrinsic::riscv_sseg4_load_mask:
13580 case Intrinsic::riscv_sseg5_load_mask:
13581 case Intrinsic::riscv_sseg6_load_mask:
13582 case Intrinsic::riscv_sseg7_load_mask:
13583 case Intrinsic::riscv_sseg8_load_mask:
13584 return lowerFixedVectorSegLoadIntrinsics(IntNo, Op, Subtarget, DAG);
13585
13586 case Intrinsic::riscv_sf_vc_v_x_se:
13587 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_X_SE);
13588 case Intrinsic::riscv_sf_vc_v_i_se:
13589 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_I_SE);
13590 case Intrinsic::riscv_sf_vc_v_xv_se:
13591 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XV_SE);
13592 case Intrinsic::riscv_sf_vc_v_iv_se:
13593 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IV_SE);
13594 case Intrinsic::riscv_sf_vc_v_vv_se:
13595 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VV_SE);
13596 case Intrinsic::riscv_sf_vc_v_fv_se:
13597 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FV_SE);
13598 case Intrinsic::riscv_sf_vc_v_xvv_se:
13599 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XVV_SE);
13600 case Intrinsic::riscv_sf_vc_v_ivv_se:
13601 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IVV_SE);
13602 case Intrinsic::riscv_sf_vc_v_vvv_se:
13603 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VVV_SE);
13604 case Intrinsic::riscv_sf_vc_v_fvv_se:
13605 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FVV_SE);
13606 case Intrinsic::riscv_sf_vc_v_xvw_se:
13607 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XVW_SE);
13608 case Intrinsic::riscv_sf_vc_v_ivw_se:
13609 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IVW_SE);
13610 case Intrinsic::riscv_sf_vc_v_vvw_se:
13611 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VVW_SE);
13612 case Intrinsic::riscv_sf_vc_v_fvw_se:
13613 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FVW_SE);
13614 }
13615
13616 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
13617}
13618
13619static SDValue
13620lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
13621 const RISCVSubtarget &Subtarget,
13622 SelectionDAG &DAG) {
13623 bool IsStrided;
13624 switch (IntNo) {
13625 case Intrinsic::riscv_seg2_store_mask:
13626 case Intrinsic::riscv_seg3_store_mask:
13627 case Intrinsic::riscv_seg4_store_mask:
13628 case Intrinsic::riscv_seg5_store_mask:
13629 case Intrinsic::riscv_seg6_store_mask:
13630 case Intrinsic::riscv_seg7_store_mask:
13631 case Intrinsic::riscv_seg8_store_mask:
13632 IsStrided = false;
13633 break;
13634 case Intrinsic::riscv_sseg2_store_mask:
13635 case Intrinsic::riscv_sseg3_store_mask:
13636 case Intrinsic::riscv_sseg4_store_mask:
13637 case Intrinsic::riscv_sseg5_store_mask:
13638 case Intrinsic::riscv_sseg6_store_mask:
13639 case Intrinsic::riscv_sseg7_store_mask:
13640 case Intrinsic::riscv_sseg8_store_mask:
13641 IsStrided = true;
13642 break;
13643 default:
13644 llvm_unreachable("unexpected intrinsic ID");
13645 }
13646
13647 SDLoc DL(Op);
13648 static const Intrinsic::ID VssegInts[] = {
13649 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
13650 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
13651 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
13652 Intrinsic::riscv_vsseg8_mask};
13653 static const Intrinsic::ID VsssegInts[] = {
13654 Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
13655 Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
13656 Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
13657 Intrinsic::riscv_vssseg8_mask};
13658
13659 // Operands: (chain, int_id, vec*, ptr, mask, vl) or
13660 // (chain, int_id, vec*, ptr, stride, mask, vl)
13661 unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
13662 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
13663 MVT XLenVT = Subtarget.getXLenVT();
13664 MVT VT = Op->getOperand(Num: 2).getSimpleValueType();
13665 MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
13666 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
13667 ContainerVT.getScalarSizeInBits();
13668 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: NF);
13669
13670 SDValue VL = Op.getOperand(i: Op.getNumOperands() - 1);
13671 SDValue Mask = Op.getOperand(i: Op.getNumOperands() - 2);
13672 MVT MaskVT = Mask.getSimpleValueType();
13673 MVT MaskContainerVT = ::getContainerForFixedLengthVector(VT: MaskVT, Subtarget);
13674 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
13675
13676 SDValue IntID = DAG.getTargetConstant(
13677 Val: IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, VT: XLenVT);
13678 SDValue Ptr = Op->getOperand(Num: NF + 2);
13679
13680 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Val&: Op);
13681
13682 SDValue StoredVal = DAG.getUNDEF(VT: VecTupTy);
13683 for (unsigned i = 0; i < NF; i++)
13684 StoredVal = DAG.getNode(
13685 Opcode: RISCVISD::TUPLE_INSERT, DL, VT: VecTupTy, N1: StoredVal,
13686 N2: convertToScalableVector(VT: ContainerVT, V: FixedIntrinsic->getOperand(Num: 2 + i),
13687 DAG, Subtarget),
13688 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
13689
13690 SmallVector<SDValue, 10> Ops = {
13691 FixedIntrinsic->getChain(),
13692 IntID,
13693 StoredVal,
13694 Ptr,
13695 Mask,
13696 VL,
13697 DAG.getTargetConstant(Val: Log2_64(Value: VT.getScalarSizeInBits()), DL, VT: XLenVT)};
13698 // Insert the stride operand.
13699 if (IsStrided)
13700 Ops.insert(I: std::next(x: Ops.begin(), n: 4),
13701 Elt: Op.getOperand(i: Op.getNumOperands() - 3));
13702
13703 return DAG.getMemIntrinsicNode(
13704 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
13705 MemVT: FixedIntrinsic->getMemoryVT(), MMO: FixedIntrinsic->getMemOperand());
13706}
13707
13708SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
13709 SelectionDAG &DAG) const {
13710 unsigned IntNo = Op.getConstantOperandVal(i: 1);
13711 switch (IntNo) {
13712 default:
13713 break;
13714 case Intrinsic::riscv_seg2_store_mask:
13715 case Intrinsic::riscv_seg3_store_mask:
13716 case Intrinsic::riscv_seg4_store_mask:
13717 case Intrinsic::riscv_seg5_store_mask:
13718 case Intrinsic::riscv_seg6_store_mask:
13719 case Intrinsic::riscv_seg7_store_mask:
13720 case Intrinsic::riscv_seg8_store_mask:
13721 case Intrinsic::riscv_sseg2_store_mask:
13722 case Intrinsic::riscv_sseg3_store_mask:
13723 case Intrinsic::riscv_sseg4_store_mask:
13724 case Intrinsic::riscv_sseg5_store_mask:
13725 case Intrinsic::riscv_sseg6_store_mask:
13726 case Intrinsic::riscv_sseg7_store_mask:
13727 case Intrinsic::riscv_sseg8_store_mask:
13728 return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);
13729
13730 case Intrinsic::riscv_sf_vc_xv_se:
13731 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XV_SE);
13732 case Intrinsic::riscv_sf_vc_iv_se:
13733 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IV_SE);
13734 case Intrinsic::riscv_sf_vc_vv_se:
13735 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VV_SE);
13736 case Intrinsic::riscv_sf_vc_fv_se:
13737 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FV_SE);
13738 case Intrinsic::riscv_sf_vc_xvv_se:
13739 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XVV_SE);
13740 case Intrinsic::riscv_sf_vc_ivv_se:
13741 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IVV_SE);
13742 case Intrinsic::riscv_sf_vc_vvv_se:
13743 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VVV_SE);
13744 case Intrinsic::riscv_sf_vc_fvv_se:
13745 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FVV_SE);
13746 case Intrinsic::riscv_sf_vc_xvw_se:
13747 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XVW_SE);
13748 case Intrinsic::riscv_sf_vc_ivw_se:
13749 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IVW_SE);
13750 case Intrinsic::riscv_sf_vc_vvw_se:
13751 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VVW_SE);
13752 case Intrinsic::riscv_sf_vc_fvw_se:
13753 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FVW_SE);
13754 }
13755
13756 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
13757}
13758
13759static unsigned getRVVReductionOp(unsigned ISDOpcode) {
13760 switch (ISDOpcode) {
13761 default:
13762 llvm_unreachable("Unhandled reduction");
13763 case ISD::VP_REDUCE_ADD:
13764 case ISD::VECREDUCE_ADD:
13765 return RISCVISD::VECREDUCE_ADD_VL;
13766 case ISD::VP_REDUCE_UMAX:
13767 case ISD::VECREDUCE_UMAX:
13768 return RISCVISD::VECREDUCE_UMAX_VL;
13769 case ISD::VP_REDUCE_SMAX:
13770 case ISD::VECREDUCE_SMAX:
13771 return RISCVISD::VECREDUCE_SMAX_VL;
13772 case ISD::VP_REDUCE_UMIN:
13773 case ISD::VECREDUCE_UMIN:
13774 return RISCVISD::VECREDUCE_UMIN_VL;
13775 case ISD::VP_REDUCE_SMIN:
13776 case ISD::VECREDUCE_SMIN:
13777 return RISCVISD::VECREDUCE_SMIN_VL;
13778 case ISD::VP_REDUCE_AND:
13779 case ISD::VECREDUCE_AND:
13780 return RISCVISD::VECREDUCE_AND_VL;
13781 case ISD::VP_REDUCE_OR:
13782 case ISD::VECREDUCE_OR:
13783 return RISCVISD::VECREDUCE_OR_VL;
13784 case ISD::VP_REDUCE_XOR:
13785 case ISD::VECREDUCE_XOR:
13786 return RISCVISD::VECREDUCE_XOR_VL;
13787 case ISD::VP_REDUCE_FADD:
13788 return RISCVISD::VECREDUCE_FADD_VL;
13789 case ISD::VP_REDUCE_SEQ_FADD:
13790 return RISCVISD::VECREDUCE_SEQ_FADD_VL;
13791 case ISD::VP_REDUCE_FMAX:
13792 case ISD::VP_REDUCE_FMAXIMUM:
13793 return RISCVISD::VECREDUCE_FMAX_VL;
13794 case ISD::VP_REDUCE_FMIN:
13795 case ISD::VP_REDUCE_FMINIMUM:
13796 return RISCVISD::VECREDUCE_FMIN_VL;
13797 }
13798
13799}
13800
13801SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
13802 SelectionDAG &DAG,
13803 bool IsVP) const {
13804 SDLoc DL(Op);
13805 SDValue Vec = Op.getOperand(i: IsVP ? 1 : 0);
13806 MVT VecVT = Vec.getSimpleValueType();
13807 assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
13808 Op.getOpcode() == ISD::VECREDUCE_OR ||
13809 Op.getOpcode() == ISD::VECREDUCE_XOR ||
13810 Op.getOpcode() == ISD::VP_REDUCE_AND ||
13811 Op.getOpcode() == ISD::VP_REDUCE_OR ||
13812 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
13813 "Unexpected reduction lowering");
13814
13815 MVT XLenVT = Subtarget.getXLenVT();
13816
13817 MVT ContainerVT = VecVT;
13818 if (VecVT.isFixedLengthVector()) {
13819 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13820 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
13821 }
13822
13823 SDValue Mask, VL;
13824 if (IsVP) {
13825 Mask = Op.getOperand(i: 2);
13826 VL = Op.getOperand(i: 3);
13827 } else {
13828 std::tie(args&: Mask, args&: VL) =
13829 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
13830 }
13831
13832 ISD::CondCode CC;
13833 switch (Op.getOpcode()) {
13834 default:
13835 llvm_unreachable("Unhandled reduction");
13836 case ISD::VECREDUCE_AND:
13837 case ISD::VP_REDUCE_AND: {
13838 // vcpop ~x == 0
13839 if (VecVT.isFixedLengthVector())
13840 Vec = DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Vec,
13841 N2: DAG.getAllOnesConstant(DL, VT: ContainerVT), N3: VL);
13842 else
13843 Vec = DAG.getNOT(DL, Val: Vec, VT: ContainerVT);
13844 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
13845 CC = ISD::SETEQ;
13846 break;
13847 }
13848 case ISD::VECREDUCE_OR:
13849 case ISD::VP_REDUCE_OR:
13850 // vcpop x != 0
13851 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
13852 CC = ISD::SETNE;
13853 break;
13854 case ISD::VECREDUCE_XOR:
13855 case ISD::VP_REDUCE_XOR: {
13856 // ((vcpop x) & 1) != 0
13857 SDValue One = DAG.getConstant(Val: 1, DL, VT: XLenVT);
13858 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
13859 Vec = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Vec, N2: One);
13860 CC = ISD::SETNE;
13861 break;
13862 }
13863 }
13864
13865 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
13866 SDValue SetCC = DAG.getSetCC(DL, VT: XLenVT, LHS: Vec, RHS: Zero, Cond: CC);
13867 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: SetCC);
13868
13869 if (!IsVP)
13870 return SetCC;
13871
13872 // Now include the start value in the operation.
13873 // Note that we must return the start value when no elements are operated
13874 // upon. The vcpop instructions we've emitted in each case above will return
13875 // 0 for an inactive vector, and so we've already received the neutral value:
13876 // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
13877 // can simply include the start value.
13878 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
13879 return DAG.getNode(Opcode: BaseOpc, DL, VT: Op.getValueType(), N1: SetCC, N2: Op.getOperand(i: 0));
13880}
13881
13882static bool isNonZeroAVL(SDValue AVL) {
13883 auto *RegisterAVL = dyn_cast<RegisterSDNode>(Val&: AVL);
13884 auto *ImmAVL = dyn_cast<ConstantSDNode>(Val&: AVL);
13885 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
13886 (ImmAVL && ImmAVL->getZExtValue() >= 1);
13887}
13888
13889/// Helper to lower a reduction sequence of the form:
13890/// scalar = reduce_op vec, scalar_start
13891static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT,
13892 SDValue StartValue, SDValue Vec, SDValue Mask,
13893 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
13894 const RISCVSubtarget &Subtarget) {
13895 const MVT VecVT = Vec.getSimpleValueType();
13896 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: VecVT);
13897 const MVT XLenVT = Subtarget.getXLenVT();
13898 const bool NonZeroAVL = isNonZeroAVL(AVL: VL);
13899
13900 // The reduction needs an LMUL1 input; do the splat at either LMUL1
13901 // or the original VT if fractional.
13902 auto InnerVT = VecVT.bitsLE(VT: M1VT) ? VecVT : M1VT;
13903 // We reuse the VL of the reduction to reduce vsetvli toggles if we can
13904 // prove it is non-zero. For the AVL=0 case, we need the scalar to
13905 // be the result of the reduction operation.
13906 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(Val: 1, DL, VT: XLenVT);
13907 SDValue InitialValue =
13908 lowerScalarInsert(Scalar: StartValue, VL: InnerVL, VT: InnerVT, DL, DAG, Subtarget);
13909 if (M1VT != InnerVT)
13910 InitialValue =
13911 DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: M1VT), SubVec: InitialValue, Idx: 0);
13912 SDValue PassThru = NonZeroAVL ? DAG.getUNDEF(VT: M1VT) : InitialValue;
13913 SDValue Policy = DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
13914 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
13915 SDValue Reduction = DAG.getNode(Opcode: RVVOpcode, DL, VT: M1VT, Ops);
13916 return DAG.getExtractVectorElt(DL, VT: ResVT, Vec: Reduction, Idx: 0);
13917}
13918
13919SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
13920 SelectionDAG &DAG) const {
13921 SDLoc DL(Op);
13922 SDValue Vec = Op.getOperand(i: 0);
13923 EVT VecEVT = Vec.getValueType();
13924
13925 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
13926
13927 // Due to ordering in legalize types we may have a vector type that needs to
13928 // be split. Do that manually so we can get down to a legal type.
13929 while (getTypeAction(Context&: *DAG.getContext(), VT: VecEVT) ==
13930 TargetLowering::TypeSplitVector) {
13931 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
13932 VecEVT = Lo.getValueType();
13933 Vec = DAG.getNode(Opcode: BaseOpc, DL, VT: VecEVT, N1: Lo, N2: Hi);
13934 }
13935
13936 // TODO: The type may need to be widened rather than split. Or widened before
13937 // it can be split.
13938 if (!isTypeLegal(VT: VecEVT))
13939 return SDValue();
13940
13941 MVT VecVT = VecEVT.getSimpleVT();
13942 MVT VecEltVT = VecVT.getVectorElementType();
13943 unsigned RVVOpcode = getRVVReductionOp(ISDOpcode: Op.getOpcode());
13944
13945 MVT ContainerVT = VecVT;
13946 if (VecVT.isFixedLengthVector()) {
13947 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13948 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
13949 }
13950
13951 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
13952
13953 SDValue StartV;
13954 switch (BaseOpc) {
13955 default:
13956 StartV = DAG.getIdentityElement(Opcode: BaseOpc, DL, VT: VecEltVT, Flags: SDNodeFlags());
13957 break;
13958 case ISD::AND:
13959 case ISD::OR:
13960 case ISD::UMAX:
13961 case ISD::UMIN:
13962 case ISD::SMAX:
13963 case ISD::SMIN:
13964 StartV = DAG.getExtractVectorElt(DL, VT: VecEltVT, Vec, Idx: 0);
13965 break;
13966 }
13967 return lowerReductionSeq(RVVOpcode, ResVT: Op.getSimpleValueType(), StartValue: StartV, Vec,
13968 Mask, VL, DL, DAG, Subtarget);
13969}
13970
13971// Given a reduction op, this function returns the matching reduction opcode,
13972// the vector SDValue and the scalar SDValue required to lower this to a
13973// RISCVISD node.
13974static std::tuple<unsigned, SDValue, SDValue>
13975getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT,
13976 const RISCVSubtarget &Subtarget) {
13977 SDLoc DL(Op);
13978 auto Flags = Op->getFlags();
13979 unsigned Opcode = Op.getOpcode();
13980 switch (Opcode) {
13981 default:
13982 llvm_unreachable("Unhandled reduction");
13983 case ISD::VECREDUCE_FADD: {
13984 // Use positive zero if we can. It is cheaper to materialize.
13985 SDValue Zero =
13986 DAG.getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT: EltVT);
13987 return std::make_tuple(args: RISCVISD::VECREDUCE_FADD_VL, args: Op.getOperand(i: 0), args&: Zero);
13988 }
13989 case ISD::VECREDUCE_SEQ_FADD:
13990 return std::make_tuple(args: RISCVISD::VECREDUCE_SEQ_FADD_VL, args: Op.getOperand(i: 1),
13991 args: Op.getOperand(i: 0));
13992 case ISD::VECREDUCE_FMINIMUM:
13993 case ISD::VECREDUCE_FMAXIMUM:
13994 case ISD::VECREDUCE_FMIN:
13995 case ISD::VECREDUCE_FMAX: {
13996 SDValue Front = DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op.getOperand(i: 0), Idx: 0);
13997 unsigned RVVOpc =
13998 (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
13999 ? RISCVISD::VECREDUCE_FMIN_VL
14000 : RISCVISD::VECREDUCE_FMAX_VL;
14001 return std::make_tuple(args&: RVVOpc, args: Op.getOperand(i: 0), args&: Front);
14002 }
14003 }
14004}
14005
14006SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
14007 SelectionDAG &DAG) const {
14008 SDLoc DL(Op);
14009 MVT VecEltVT = Op.getSimpleValueType();
14010
14011 unsigned RVVOpcode;
14012 SDValue VectorVal, ScalarVal;
14013 std::tie(args&: RVVOpcode, args&: VectorVal, args&: ScalarVal) =
14014 getRVVFPReductionOpAndOperands(Op, DAG, EltVT: VecEltVT, Subtarget);
14015 MVT VecVT = VectorVal.getSimpleValueType();
14016
14017 MVT ContainerVT = VecVT;
14018 if (VecVT.isFixedLengthVector()) {
14019 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
14020 VectorVal = convertToScalableVector(VT: ContainerVT, V: VectorVal, DAG, Subtarget);
14021 }
14022
14023 MVT ResVT = Op.getSimpleValueType();
14024 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
14025 SDValue Res = lowerReductionSeq(RVVOpcode, ResVT, StartValue: ScalarVal, Vec: VectorVal, Mask,
14026 VL, DL, DAG, Subtarget);
14027 if (Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
14028 Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
14029 return Res;
14030
14031 if (Op->getFlags().hasNoNaNs())
14032 return Res;
14033
14034 // Force output to NaN if any element is Nan.
14035 SDValue IsNan =
14036 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
14037 Ops: {VectorVal, VectorVal, DAG.getCondCode(Cond: ISD::SETNE),
14038 DAG.getUNDEF(VT: Mask.getValueType()), Mask, VL});
14039 MVT XLenVT = Subtarget.getXLenVT();
14040 SDValue CPop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: IsNan, N2: Mask, N3: VL);
14041 SDValue NoNaNs = DAG.getSetCC(DL, VT: XLenVT, LHS: CPop,
14042 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
14043 return DAG.getSelect(
14044 DL, VT: ResVT, Cond: NoNaNs, LHS: Res,
14045 RHS: DAG.getConstantFP(Val: APFloat::getNaN(Sem: ResVT.getFltSemantics()), DL, VT: ResVT));
14046}
14047
14048SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
14049 SelectionDAG &DAG) const {
14050 SDLoc DL(Op);
14051 unsigned Opc = Op.getOpcode();
14052 SDValue Start = Op.getOperand(i: 0);
14053 SDValue Vec = Op.getOperand(i: 1);
14054 EVT VecEVT = Vec.getValueType();
14055 MVT XLenVT = Subtarget.getXLenVT();
14056
14057 // TODO: The type may need to be widened rather than split. Or widened before
14058 // it can be split.
14059 if (!isTypeLegal(VT: VecEVT))
14060 return SDValue();
14061
14062 MVT VecVT = VecEVT.getSimpleVT();
14063 unsigned RVVOpcode = getRVVReductionOp(ISDOpcode: Opc);
14064
14065 if (VecVT.isFixedLengthVector()) {
14066 auto ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
14067 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
14068 }
14069
14070 SDValue VL = Op.getOperand(i: 3);
14071 SDValue Mask = Op.getOperand(i: 2);
14072 SDValue Res =
14073 lowerReductionSeq(RVVOpcode, ResVT: Op.getSimpleValueType(), StartValue: Op.getOperand(i: 0),
14074 Vec, Mask, VL, DL, DAG, Subtarget);
14075 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
14076 Op->getFlags().hasNoNaNs())
14077 return Res;
14078
14079 // Propagate NaNs.
14080 MVT PredVT = getMaskTypeFor(VecVT: Vec.getSimpleValueType());
14081 // Check if any of the elements in Vec is NaN.
14082 SDValue IsNaN = DAG.getNode(
14083 Opcode: RISCVISD::SETCC_VL, DL, VT: PredVT,
14084 Ops: {Vec, Vec, DAG.getCondCode(Cond: ISD::SETNE), DAG.getUNDEF(VT: PredVT), Mask, VL});
14085 SDValue VCPop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: IsNaN, N2: Mask, N3: VL);
14086 // Check if the start value is NaN.
14087 SDValue StartIsNaN = DAG.getSetCC(DL, VT: XLenVT, LHS: Start, RHS: Start, Cond: ISD::SETUO);
14088 VCPop = DAG.getNode(Opcode: ISD::OR, DL, VT: XLenVT, N1: VCPop, N2: StartIsNaN);
14089 SDValue NoNaNs = DAG.getSetCC(DL, VT: XLenVT, LHS: VCPop,
14090 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
14091 MVT ResVT = Res.getSimpleValueType();
14092 return DAG.getSelect(
14093 DL, VT: ResVT, Cond: NoNaNs, LHS: Res,
14094 RHS: DAG.getConstantFP(Val: APFloat::getNaN(Sem: ResVT.getFltSemantics()), DL, VT: ResVT));
14095}
14096
14097static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
14098 SDValue V, MVT WideVT);
14099
14100SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
14101 SelectionDAG &DAG) const {
14102 SDValue Vec = Op.getOperand(i: 0);
14103 SDValue SubVec = Op.getOperand(i: 1);
14104 MVT VecVT = Vec.getSimpleValueType();
14105 MVT SubVecVT = SubVec.getSimpleValueType();
14106
14107 SDLoc DL(Op);
14108 MVT XLenVT = Subtarget.getXLenVT();
14109 unsigned OrigIdx = Op.getConstantOperandVal(i: 2);
14110 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
14111
14112 bool IsPExtInsert =
14113 Subtarget.hasStdExtP() &&
14114 ((Subtarget.is64Bit() &&
14115 (SubVecVT == MVT::v2i16 || SubVecVT == MVT::v4i8)) ||
14116 (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)));
14117
14118 // Fold insert of a 32-bit packed type into a zero-filled 64-bit packed vector
14119 // at index 0 (a zero-extend) to avoid scalarizing it into a byte-wise repack.
14120 if (IsPExtInsert) {
14121 if ((VecVT != MVT::v4i16 && VecVT != MVT::v8i8) ||
14122 SubVecVT.getSizeInBits() != 32 || OrigIdx != 0 ||
14123 !ISD::isConstantSplatVectorAllZeros(N: Vec.getNode()))
14124 return SDValue();
14125
14126 if (!Subtarget.is64Bit()) {
14127 SDValue Zero = DAG.getBitcast(VT: SubVecVT, V: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
14128 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: SubVec, N2: Zero);
14129 }
14130 return widenPackedVectorWithZeros(DAG, DL, V: SubVec, WideVT: VecVT);
14131 }
14132
14133 if (OrigIdx == 0 && Vec.isUndef())
14134 return Op;
14135
14136 // We don't have the ability to slide mask vectors up indexed by their i1
14137 // elements; the smallest we can do is i8. Often we are able to bitcast to
14138 // equivalent i8 vectors. Note that when inserting a fixed-length vector
14139 // into a scalable one, we might not necessarily have enough scalable
14140 // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
14141 if (SubVecVT.getVectorElementType() == MVT::i1) {
14142 if (VecVT.getVectorMinNumElements() >= 8 &&
14143 SubVecVT.getVectorMinNumElements() >= 8) {
14144 assert(OrigIdx % 8 == 0 && "Invalid index");
14145 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
14146 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
14147 "Unexpected mask vector lowering");
14148 OrigIdx /= 8;
14149 SubVecVT =
14150 MVT::getVectorVT(VT: MVT::i8, NumElements: SubVecVT.getVectorMinNumElements() / 8,
14151 IsScalable: SubVecVT.isScalableVector());
14152 VecVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VecVT.getVectorMinNumElements() / 8,
14153 IsScalable: VecVT.isScalableVector());
14154 Vec = DAG.getBitcast(VT: VecVT, V: Vec);
14155 SubVec = DAG.getBitcast(VT: SubVecVT, V: SubVec);
14156 } else {
14157 // We can't slide this mask vector up indexed by its i1 elements.
14158 // This poses a problem when we wish to insert a scalable vector which
14159 // can't be re-expressed as a larger type. Just choose the slow path and
14160 // extend to a larger type, then truncate back down.
14161 MVT ExtVecVT = VecVT.changeVectorElementType(EltVT: MVT::i8);
14162 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(EltVT: MVT::i8);
14163 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVecVT, Operand: Vec);
14164 SubVec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtSubVecVT, Operand: SubVec);
14165 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ExtVecVT, N1: Vec, N2: SubVec,
14166 N3: Op.getOperand(i: 2));
14167 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: ExtVecVT);
14168 return DAG.getSetCC(DL, VT: VecVT, LHS: Vec, RHS: SplatZero, Cond: ISD::SETNE);
14169 }
14170 }
14171
14172 // If the subvector vector is a fixed-length type and we don't know VLEN
14173 // exactly, we cannot use subregister manipulation to simplify the codegen; we
14174 // don't know which register of a LMUL group contains the specific subvector
14175 // as we only know the minimum register size. Therefore we must slide the
14176 // vector group up the full amount.
14177 const auto VLen = Subtarget.getRealVLen();
14178 if (SubVecVT.isFixedLengthVector() && !VLen) {
14179 MVT ContainerVT = VecVT;
14180 if (VecVT.isFixedLengthVector()) {
14181 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
14182 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
14183 }
14184
14185 SubVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec, Idx: 0);
14186
14187 SDValue Mask =
14188 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
14189 // Set the vector length to only the number of elements we care about. Note
14190 // that for slideup this includes the offset.
14191 unsigned EndIndex = OrigIdx + SubVecVT.getVectorNumElements();
14192 SDValue VL = DAG.getConstant(Val: EndIndex, DL, VT: XLenVT);
14193
14194 // Use tail agnostic policy if we're inserting over Vec's tail.
14195 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
14196 if (VecVT.isFixedLengthVector() && EndIndex == VecVT.getVectorNumElements())
14197 Policy = RISCVVType::TAIL_AGNOSTIC;
14198
14199 // If we're inserting into the lowest elements, use a tail undisturbed
14200 // vmv.v.v.
14201 if (OrigIdx == 0) {
14202 SubVec =
14203 DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: ContainerVT, N1: Vec, N2: SubVec, N3: VL);
14204 } else {
14205 SDValue SlideupAmt = DAG.getConstant(Val: OrigIdx, DL, VT: XLenVT);
14206 SubVec = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Vec, Op: SubVec,
14207 Offset: SlideupAmt, Mask, VL, Policy);
14208 }
14209
14210 if (VecVT.isFixedLengthVector())
14211 SubVec = convertFromScalableVector(VT: VecVT, V: SubVec, DAG, Subtarget);
14212 return DAG.getBitcast(VT: Op.getValueType(), V: SubVec);
14213 }
14214
14215 MVT ContainerVecVT = VecVT;
14216 if (VecVT.isFixedLengthVector()) {
14217 ContainerVecVT = getContainerForFixedLengthVector(VT: VecVT);
14218 Vec = convertToScalableVector(VT: ContainerVecVT, V: Vec, DAG, Subtarget);
14219 }
14220
14221 MVT ContainerSubVecVT = SubVecVT;
14222 if (SubVecVT.isFixedLengthVector()) {
14223 ContainerSubVecVT = getContainerForFixedLengthVector(VT: SubVecVT);
14224 SubVec = convertToScalableVector(VT: ContainerSubVecVT, V: SubVec, DAG, Subtarget);
14225 }
14226
14227 unsigned SubRegIdx;
14228 ElementCount RemIdx;
14229 // insert_subvector scales the index by vscale if the subvector is scalable,
14230 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
14231 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
14232 if (SubVecVT.isFixedLengthVector()) {
14233 assert(VLen);
14234 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
14235 auto Decompose =
14236 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
14237 VecVT: ContainerVecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx / Vscale, TRI);
14238 SubRegIdx = Decompose.first;
14239 RemIdx = ElementCount::getFixed(MinVal: (Decompose.second * Vscale) +
14240 (OrigIdx % Vscale));
14241 } else {
14242 auto Decompose =
14243 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
14244 VecVT: ContainerVecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx, TRI);
14245 SubRegIdx = Decompose.first;
14246 RemIdx = ElementCount::getScalable(MinVal: Decompose.second);
14247 }
14248
14249 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
14250 assert(isPowerOf2_64(
14251 Subtarget.expandVScale(SubVecVT.getSizeInBits()).getKnownMinValue()));
14252 bool ExactlyVecRegSized =
14253 Subtarget.expandVScale(X: SubVecVT.getSizeInBits())
14254 .isKnownMultipleOf(RHS: Subtarget.expandVScale(X: VecRegSize));
14255
14256 // 1. If the Idx has been completely eliminated and this subvector's size is
14257 // a vector register or a multiple thereof, or the surrounding elements are
14258 // undef, then this is a subvector insert which naturally aligns to a vector
14259 // register. These can easily be handled using subregister manipulation.
14260 // 2. If the subvector isn't an exact multiple of a valid register group size,
14261 // then the insertion must preserve the undisturbed elements of the register.
14262 // We do this by lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1
14263 // vector type (which resolves to a subregister copy), performing a VSLIDEUP
14264 // to place the subvector within the vector register, and an INSERT_SUBVECTOR
14265 // of that LMUL=1 type back into the larger vector (resolving to another
14266 // subregister operation). See below for how our VSLIDEUP works. We go via a
14267 // LMUL=1 type to avoid allocating a large register group to hold our
14268 // subvector.
14269 if (RemIdx.isZero() && (ExactlyVecRegSized || Vec.isUndef())) {
14270 if (SubVecVT.isFixedLengthVector()) {
14271 // We may get NoSubRegister if inserting at index 0 and the subvec
14272 // container is the same as the vector, e.g. vec=v4i32,subvec=v4i32,idx=0
14273 if (SubRegIdx == RISCV::NoSubRegister) {
14274 assert(OrigIdx == 0);
14275 return Op;
14276 }
14277
14278 // Use a insert_subvector that will resolve to an insert subreg.
14279 assert(VLen);
14280 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
14281 SDValue Insert =
14282 DAG.getInsertSubvector(DL, Vec, SubVec, Idx: OrigIdx / Vscale);
14283 if (VecVT.isFixedLengthVector())
14284 Insert = convertFromScalableVector(VT: VecVT, V: Insert, DAG, Subtarget);
14285 return Insert;
14286 }
14287 return Op;
14288 }
14289
14290 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
14291 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
14292 // (in our case undisturbed). This means we can set up a subvector insertion
14293 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
14294 // size of the subvector.
14295 MVT InterSubVT = ContainerVecVT;
14296 SDValue AlignedExtract = Vec;
14297 unsigned AlignedIdx = OrigIdx - RemIdx.getKnownMinValue();
14298 if (SubVecVT.isFixedLengthVector()) {
14299 assert(VLen);
14300 AlignedIdx /= *VLen / RISCV::RVVBitsPerBlock;
14301 }
14302 if (ContainerVecVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVecVT))) {
14303 InterSubVT = RISCVTargetLowering::getM1VT(VT: ContainerVecVT);
14304 // Extract a subvector equal to the nearest full vector register type. This
14305 // should resolve to a EXTRACT_SUBREG instruction.
14306 AlignedExtract = DAG.getExtractSubvector(DL, VT: InterSubVT, Vec, Idx: AlignedIdx);
14307 }
14308
14309 SubVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: InterSubVT), SubVec, Idx: 0);
14310
14311 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: ContainerVecVT, DL, DAG, Subtarget);
14312
14313 ElementCount EndIndex = RemIdx + SubVecVT.getVectorElementCount();
14314 VL = DAG.getElementCount(DL, VT: XLenVT, EC: SubVecVT.getVectorElementCount());
14315
14316 // Use tail agnostic policy if we're inserting over InterSubVT's tail.
14317 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
14318 if (Subtarget.expandVScale(X: EndIndex) ==
14319 Subtarget.expandVScale(X: InterSubVT.getVectorElementCount()))
14320 Policy = RISCVVType::TAIL_AGNOSTIC;
14321
14322 // If we're inserting into the lowest elements, use a tail undisturbed
14323 // vmv.v.v.
14324 if (RemIdx.isZero()) {
14325 SubVec = DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: InterSubVT, N1: AlignedExtract,
14326 N2: SubVec, N3: VL);
14327 } else {
14328 SDValue SlideupAmt = DAG.getElementCount(DL, VT: XLenVT, EC: RemIdx);
14329
14330 // Construct the vector length corresponding to RemIdx + length(SubVecVT).
14331 VL = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: SlideupAmt, N2: VL);
14332
14333 SubVec = getVSlideup(DAG, Subtarget, DL, VT: InterSubVT, Passthru: AlignedExtract, Op: SubVec,
14334 Offset: SlideupAmt, Mask, VL, Policy);
14335 }
14336
14337 // If required, insert this subvector back into the correct vector register.
14338 // This should resolve to an INSERT_SUBREG instruction.
14339 if (ContainerVecVT.bitsGT(VT: InterSubVT))
14340 SubVec = DAG.getInsertSubvector(DL, Vec, SubVec, Idx: AlignedIdx);
14341
14342 if (VecVT.isFixedLengthVector())
14343 SubVec = convertFromScalableVector(VT: VecVT, V: SubVec, DAG, Subtarget);
14344
14345 // We might have bitcast from a mask type: cast back to the original type if
14346 // required.
14347 return DAG.getBitcast(VT: Op.getSimpleValueType(), V: SubVec);
14348}
14349
14350SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
14351 SelectionDAG &DAG) const {
14352 SDValue Vec = Op.getOperand(i: 0);
14353 MVT SubVecVT = Op.getSimpleValueType();
14354 MVT VecVT = Vec.getSimpleValueType();
14355
14356 SDLoc DL(Op);
14357 MVT XLenVT = Subtarget.getXLenVT();
14358 unsigned OrigIdx = Op.getConstantOperandVal(i: 1);
14359 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
14360
14361 // With an index of 0 this is a cast-like subvector, which can be performed
14362 // with subregister operations.
14363 if (OrigIdx == 0)
14364 return Op;
14365
14366 // We don't have the ability to slide mask vectors down indexed by their i1
14367 // elements; the smallest we can do is i8. Often we are able to bitcast to
14368 // equivalent i8 vectors. Note that when extracting a fixed-length vector
14369 // from a scalable one, we might not necessarily have enough scalable
14370 // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
14371 if (SubVecVT.getVectorElementType() == MVT::i1) {
14372 if (VecVT.getVectorMinNumElements() >= 8 &&
14373 SubVecVT.getVectorMinNumElements() >= 8) {
14374 assert(OrigIdx % 8 == 0 && "Invalid index");
14375 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
14376 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
14377 "Unexpected mask vector lowering");
14378 OrigIdx /= 8;
14379 SubVecVT =
14380 MVT::getVectorVT(VT: MVT::i8, NumElements: SubVecVT.getVectorMinNumElements() / 8,
14381 IsScalable: SubVecVT.isScalableVector());
14382 VecVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VecVT.getVectorMinNumElements() / 8,
14383 IsScalable: VecVT.isScalableVector());
14384 Vec = DAG.getBitcast(VT: VecVT, V: Vec);
14385 } else {
14386 // We can't slide this mask vector down, indexed by its i1 elements.
14387 // This poses a problem when we wish to extract a scalable vector which
14388 // can't be re-expressed as a larger type. Just choose the slow path and
14389 // extend to a larger type, then truncate back down.
14390 // TODO: We could probably improve this when extracting certain fixed
14391 // from fixed, where we can extract as i8 and shift the correct element
14392 // right to reach the desired subvector?
14393 MVT ExtVecVT = VecVT.changeVectorElementType(EltVT: MVT::i8);
14394 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(EltVT: MVT::i8);
14395 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVecVT, Operand: Vec);
14396 Vec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ExtSubVecVT, N1: Vec,
14397 N2: Op.getOperand(i: 1));
14398 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: ExtSubVecVT);
14399 return DAG.getSetCC(DL, VT: SubVecVT, LHS: Vec, RHS: SplatZero, Cond: ISD::SETNE);
14400 }
14401 }
14402
14403 const auto VLen = Subtarget.getRealVLen();
14404
14405 // If the subvector vector is a fixed-length type and we don't know VLEN
14406 // exactly, we cannot use subregister manipulation to simplify the codegen; we
14407 // don't know which register of a LMUL group contains the specific subvector
14408 // as we only know the minimum register size. Therefore we must slide the
14409 // vector group down the full amount.
14410 if (SubVecVT.isFixedLengthVector() && !VLen) {
14411 MVT ContainerVT = VecVT;
14412 if (VecVT.isFixedLengthVector()) {
14413 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
14414 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
14415 }
14416
14417 // Shrink down Vec so we're performing the slidedown on a smaller LMUL.
14418 unsigned LastIdx = OrigIdx + SubVecVT.getVectorNumElements() - 1;
14419 if (auto ShrunkVT =
14420 getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: LastIdx, DL, DAG, Subtarget)) {
14421 ContainerVT = *ShrunkVT;
14422 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: 0);
14423 }
14424
14425 SDValue Mask =
14426 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
14427 // Set the vector length to only the number of elements we care about. This
14428 // avoids sliding down elements we're going to discard straight away.
14429 SDValue VL = DAG.getConstant(Val: SubVecVT.getVectorNumElements(), DL, VT: XLenVT);
14430 SDValue SlidedownAmt = DAG.getConstant(Val: OrigIdx, DL, VT: XLenVT);
14431 SDValue Slidedown =
14432 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
14433 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: SlidedownAmt, Mask, VL);
14434 // Now we can use a cast-like subvector extract to get the result.
14435 Slidedown = DAG.getExtractSubvector(DL, VT: SubVecVT, Vec: Slidedown, Idx: 0);
14436 return DAG.getBitcast(VT: Op.getValueType(), V: Slidedown);
14437 }
14438
14439 if (VecVT.isFixedLengthVector()) {
14440 VecVT = getContainerForFixedLengthVector(VT: VecVT);
14441 Vec = convertToScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
14442 }
14443
14444 MVT ContainerSubVecVT = SubVecVT;
14445 if (SubVecVT.isFixedLengthVector())
14446 ContainerSubVecVT = getContainerForFixedLengthVector(VT: SubVecVT);
14447
14448 unsigned SubRegIdx;
14449 ElementCount RemIdx;
14450 // extract_subvector scales the index by vscale if the subvector is scalable,
14451 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
14452 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
14453 if (SubVecVT.isFixedLengthVector()) {
14454 assert(VLen);
14455 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
14456 auto Decompose =
14457 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
14458 VecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx / Vscale, TRI);
14459 SubRegIdx = Decompose.first;
14460 RemIdx = ElementCount::getFixed(MinVal: (Decompose.second * Vscale) +
14461 (OrigIdx % Vscale));
14462 } else {
14463 auto Decompose =
14464 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
14465 VecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx, TRI);
14466 SubRegIdx = Decompose.first;
14467 RemIdx = ElementCount::getScalable(MinVal: Decompose.second);
14468 }
14469
14470 // If the Idx has been completely eliminated then this is a subvector extract
14471 // which naturally aligns to a vector register. These can easily be handled
14472 // using subregister manipulation. We use an extract_subvector that will
14473 // resolve to an extract subreg.
14474 if (RemIdx.isZero()) {
14475 if (SubVecVT.isFixedLengthVector()) {
14476 assert(VLen);
14477 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
14478 Vec =
14479 DAG.getExtractSubvector(DL, VT: ContainerSubVecVT, Vec, Idx: OrigIdx / Vscale);
14480 return convertFromScalableVector(VT: SubVecVT, V: Vec, DAG, Subtarget);
14481 }
14482 return Op;
14483 }
14484
14485 // Else SubVecVT is M1 or smaller and may need to be slid down: if SubVecVT
14486 // was > M1 then the index would need to be a multiple of VLMAX, and so would
14487 // divide exactly.
14488 assert(RISCVVType::decodeVLMUL(getLMUL(ContainerSubVecVT)).second ||
14489 getLMUL(ContainerSubVecVT) == RISCVVType::LMUL_1);
14490
14491 // If the vector type is an LMUL-group type, extract a subvector equal to the
14492 // nearest full vector register type.
14493 MVT InterSubVT = VecVT;
14494 if (VecVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: VecVT))) {
14495 // If VecVT has an LMUL > 1, then SubVecVT should have a smaller LMUL, and
14496 // we should have successfully decomposed the extract into a subregister.
14497 // We use an extract_subvector that will resolve to a subreg extract.
14498 assert(SubRegIdx != RISCV::NoSubRegister);
14499 (void)SubRegIdx;
14500 unsigned Idx = OrigIdx - RemIdx.getKnownMinValue();
14501 if (SubVecVT.isFixedLengthVector()) {
14502 assert(VLen);
14503 Idx /= *VLen / RISCV::RVVBitsPerBlock;
14504 }
14505 InterSubVT = RISCVTargetLowering::getM1VT(VT: VecVT);
14506 Vec = DAG.getExtractSubvector(DL, VT: InterSubVT, Vec, Idx);
14507 }
14508
14509 // Slide this vector register down by the desired number of elements in order
14510 // to place the desired subvector starting at element 0.
14511 SDValue SlidedownAmt = DAG.getElementCount(DL, VT: XLenVT, EC: RemIdx);
14512 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: InterSubVT, DL, DAG, Subtarget);
14513 if (SubVecVT.isFixedLengthVector())
14514 VL = DAG.getConstant(Val: SubVecVT.getVectorNumElements(), DL, VT: XLenVT);
14515 SDValue Slidedown =
14516 getVSlidedown(DAG, Subtarget, DL, VT: InterSubVT, Passthru: DAG.getUNDEF(VT: InterSubVT),
14517 Op: Vec, Offset: SlidedownAmt, Mask, VL);
14518
14519 // Now the vector is in the right position, extract our final subvector. This
14520 // should resolve to a COPY.
14521 Slidedown = DAG.getExtractSubvector(DL, VT: SubVecVT, Vec: Slidedown, Idx: 0);
14522
14523 // We might have bitcast from a mask type: cast back to the original type if
14524 // required.
14525 return DAG.getBitcast(VT: Op.getSimpleValueType(), V: Slidedown);
14526}
14527
14528// Widen a vector's operands to i8, then truncate its results back to the
14529// original type, typically i1. All operand and result types must be the same.
14530static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL,
14531 SelectionDAG &DAG) {
14532 MVT VT = N.getSimpleValueType();
14533 MVT WideVT = VT.changeVectorElementType(EltVT: MVT::i8);
14534 SmallVector<SDValue, 4> WideOps;
14535 for (SDValue Op : N->ops()) {
14536 assert(Op.getSimpleValueType() == VT &&
14537 "Operands and result must be same type");
14538 WideOps.push_back(Elt: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Op));
14539 }
14540
14541 unsigned NumVals = N->getNumValues();
14542
14543 SDVTList VTs = DAG.getVTList(VTs: SmallVector<EVT, 4>(
14544 NumVals,
14545 N.getValueType().changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i8)));
14546 SDValue WideN = DAG.getNode(Opcode: N.getOpcode(), DL, VTList: VTs, Ops: WideOps);
14547 SmallVector<SDValue, 4> TruncVals;
14548 for (unsigned I = 0; I < NumVals; I++) {
14549 TruncVals.push_back(
14550 Elt: DAG.getSetCC(DL, VT: N->getSimpleValueType(ResNo: I), LHS: WideN.getValue(R: I),
14551 RHS: DAG.getConstant(Val: 0, DL, VT: WideVT), Cond: ISD::SETNE));
14552 }
14553
14554 if (TruncVals.size() > 1)
14555 return DAG.getMergeValues(Ops: TruncVals, dl: DL);
14556 return TruncVals.front();
14557}
14558
14559SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
14560 SelectionDAG &DAG) const {
14561 SDLoc DL(Op);
14562 MVT VecVT = Op.getSimpleValueType();
14563
14564 const unsigned Factor = Op->getNumValues();
14565 assert(Factor <= 8);
14566
14567 // 1 bit element vectors need to be widened to e8
14568 if (VecVT.getVectorElementType() == MVT::i1)
14569 return widenVectorOpsToi8(N: Op, DL, DAG);
14570
14571 bool IsFixedVector = VecVT.isFixedLengthVector();
14572
14573 MVT ContainerVecVT = VecVT;
14574 if (IsFixedVector)
14575 ContainerVecVT = getContainerForFixedLengthVector(VT: VecVT);
14576
14577 // If concatenating would exceed LMUL=8, we need to split.
14578 if ((ContainerVecVT.getSizeInBits().getKnownMinValue() * Factor) >
14579 (8 * RISCV::RVVBitsPerBlock)) {
14580 SmallVector<SDValue, 8> Ops(Factor * 2);
14581 for (unsigned i = 0; i != Factor; ++i) {
14582 auto [OpLo, OpHi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: i);
14583 Ops[i * 2] = OpLo;
14584 Ops[i * 2 + 1] = OpHi;
14585 }
14586
14587 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
14588
14589 SDValue Lo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
14590 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
14591 SDValue Hi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
14592 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
14593
14594 SmallVector<SDValue, 8> Res(Factor);
14595 for (unsigned i = 0; i != Factor; ++i)
14596 Res[i] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo.getValue(R: i),
14597 N2: Hi.getValue(R: i));
14598
14599 return DAG.getMergeValues(Ops: Res, dl: DL);
14600 }
14601
14602 if (Subtarget.hasStdExtZvzip() && Factor == 2 && !IsFixedVector) {
14603 MVT VT = Op->getSimpleValueType(ResNo: 0);
14604 MVT NewVT = VT.getDoubleNumVectorElementsVT();
14605 if (isTypeLegal(VT: NewVT) &&
14606 isLegalVTForZvzipDeinterleavedOperand(VT, Subtarget)) {
14607 SDValue V1 = Op->getOperand(Num: 0);
14608 SDValue V2 = Op->getOperand(Num: 1);
14609 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, N1: V1, N2: V2);
14610 SDValue Even =
14611 lowerZvzipVUNZIP(Opc: RISCVISD::VUNZIPE_VL, Op: V, DL, DAG, Subtarget);
14612 SDValue Odd =
14613 lowerZvzipVUNZIP(Opc: RISCVISD::VUNZIPO_VL, Op: V, DL, DAG, Subtarget);
14614 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
14615 }
14616 }
14617
14618 SmallVector<SDValue, 8> Ops(Op->op_values());
14619
14620 // Concatenate the vectors as one vector to deinterleave
14621 MVT ConcatVT =
14622 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
14623 EC: VecVT.getVectorElementCount() * PowerOf2Ceil(A: Factor));
14624 if (Ops.size() < PowerOf2Ceil(A: Factor))
14625 Ops.append(NumInputs: PowerOf2Ceil(A: Factor) - Factor, Elt: DAG.getUNDEF(VT: VecVT));
14626 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops);
14627
14628 if (Factor == 2 && !IsFixedVector) {
14629 // We can deinterleave through vnsrl.wi if the element type is smaller than
14630 // ELEN
14631 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
14632 SDValue Even = getDeinterleaveShiftAndTrunc(DL, VT: VecVT, Src: Concat, Factor: 2, Index: 0, DAG);
14633 SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VT: VecVT, Src: Concat, Factor: 2, Index: 1, DAG);
14634 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
14635 }
14636
14637 // For the indices, use the vmv.v.x of an i8 constant to fill the largest
14638 // possibly mask vector, then extract the required subvector. Doing this
14639 // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
14640 // creation to be rematerialized during register allocation to reduce
14641 // register pressure if needed.
14642
14643 MVT MaskVT = ConcatVT.changeVectorElementType(EltVT: MVT::i1);
14644
14645 SDValue EvenSplat = DAG.getConstant(Val: 0b01010101, DL, VT: MVT::nxv8i8);
14646 EvenSplat = DAG.getBitcast(VT: MVT::nxv64i1, V: EvenSplat);
14647 SDValue EvenMask = DAG.getExtractSubvector(DL, VT: MaskVT, Vec: EvenSplat, Idx: 0);
14648
14649 SDValue OddSplat = DAG.getConstant(Val: 0b10101010, DL, VT: MVT::nxv8i8);
14650 OddSplat = DAG.getBitcast(VT: MVT::nxv64i1, V: OddSplat);
14651 SDValue OddMask = DAG.getExtractSubvector(DL, VT: MaskVT, Vec: OddSplat, Idx: 0);
14652
14653 // vcompress the even and odd elements into two separate vectors
14654 SDValue EvenWide = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: ConcatVT, N1: Concat,
14655 N2: EvenMask, N3: DAG.getUNDEF(VT: ConcatVT));
14656 SDValue OddWide = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: ConcatVT, N1: Concat,
14657 N2: OddMask, N3: DAG.getUNDEF(VT: ConcatVT));
14658
14659 // Extract the result half of the gather for even and odd
14660 SDValue Even = DAG.getExtractSubvector(DL, VT: VecVT, Vec: EvenWide, Idx: 0);
14661 SDValue Odd = DAG.getExtractSubvector(DL, VT: VecVT, Vec: OddWide, Idx: 0);
14662
14663 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
14664 }
14665
14666 // Store with unit-stride store and load it back with segmented load.
14667 SDValue Mask, VL;
14668 MVT XLenVT = Subtarget.getXLenVT();
14669 auto &MF = DAG.getMachineFunction();
14670 SDValue Chain = DAG.getEntryNode();
14671 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
14672 SDValue StackPtr;
14673 MachinePointerInfo PtrInfo;
14674 if (IsFixedVector) {
14675 // Calculating the stack size.
14676 ElementCount ActualConcatEC = VecVT.getVectorElementCount() * Factor;
14677 EVT ConcatEVT = EVT::getVectorVT(
14678 Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), EC: ActualConcatEC);
14679 StackPtr = DAG.CreateStackTemporary(Bytes: ConcatEVT.getStoreSize(), Alignment);
14680 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
14681 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
14682
14683 // If this is a fixed vector, instead of using the concat vector, we simply
14684 // store each fixed vector operand directly onto the stack, individually.
14685 // The reason being that if the fixed vector is (much) smaller than the
14686 // container vector, we will be wasting space on stack.
14687 TypeSize VecSize = VecVT.getStoreSize();
14688 SDValue BasePtr = StackPtr;
14689 MachinePointerInfo PI = PtrInfo;
14690 SmallVector<SDValue, 8> Tokens(Factor);
14691 for (auto [Idx, FieldOp] : enumerate(First: Op->op_values())) {
14692 if (Idx) {
14693 // Advance the pointer.
14694 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: VecSize);
14695 PI = PI.getWithOffset(O: VecSize);
14696 }
14697 Tokens[Idx] = DAG.getStore(Chain, dl: DL, Val: FieldOp, Ptr: BasePtr, PtrInfo: PI, Alignment);
14698 }
14699 Chain = DAG.getTokenFactor(DL, Vals&: Tokens);
14700
14701 // Calculating Mask and VL for later usages.
14702 std::tie(args&: Mask, args&: VL) =
14703 getDefaultVLOps(VecVT, ContainerVT: ContainerVecVT, DL, DAG, Subtarget);
14704 ConcatVT = getContainerForFixedLengthVector(VT: ConcatVT);
14705 } else {
14706 std::tie(args&: Mask, args&: VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
14707 StackPtr = DAG.CreateStackTemporary(Bytes: ConcatVT.getStoreSize(), Alignment);
14708 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
14709 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
14710
14711 SDValue StoreOps[] = {
14712 Chain, DAG.getTargetConstant(Val: Intrinsic::riscv_vse, DL, VT: XLenVT), Concat,
14713 StackPtr, VL};
14714
14715 Chain = DAG.getMemIntrinsicNode(
14716 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
14717 MemVT: ConcatVT.getVectorElementType(), PtrInfo, Alignment,
14718 Flags: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer());
14719 }
14720
14721 // Load it back with segmented load.
14722 SDValue Passthru = DAG.getUNDEF(VT: ConcatVT);
14723 static const Intrinsic::ID VlsegIntrinsicsIds[] = {
14724 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
14725 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
14726 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
14727 Intrinsic::riscv_vlseg8_mask};
14728
14729 SDValue LoadOps[] = {
14730 Chain,
14731 DAG.getTargetConstant(Val: VlsegIntrinsicsIds[Factor - 2], DL, VT: XLenVT),
14732 Passthru,
14733 StackPtr,
14734 Mask,
14735 VL,
14736 DAG.getTargetConstant(
14737 Val: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, VT: XLenVT),
14738 DAG.getTargetConstant(Val: Log2_64(Value: VecVT.getScalarSizeInBits()), DL, VT: XLenVT)};
14739
14740 unsigned Sz = Factor * ContainerVecVT.getVectorMinNumElements() *
14741 ContainerVecVT.getScalarSizeInBits();
14742 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: Factor);
14743
14744 SDValue Load = DAG.getMemIntrinsicNode(
14745 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: DAG.getVTList(VTs: {VecTupTy, MVT::Other}),
14746 Ops: LoadOps, MemVT: ConcatVT.getVectorElementType(), PtrInfo, Alignment,
14747 Flags: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer());
14748
14749 SmallVector<SDValue, 8> Res(Factor);
14750
14751 for (unsigned i = 0U; i < Factor; ++i) {
14752 SDValue FieldRes =
14753 DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: ContainerVecVT, N1: Load,
14754 N2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
14755 if (IsFixedVector)
14756 FieldRes = convertFromScalableVector(VT: VecVT, V: FieldRes, DAG, Subtarget);
14757 Res[i] = FieldRes;
14758 }
14759
14760 return DAG.getMergeValues(Ops: Res, dl: DL);
14761}
14762
14763SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
14764 SelectionDAG &DAG) const {
14765 SDLoc DL(Op);
14766 MVT VecVT = Op.getSimpleValueType();
14767
14768 const unsigned Factor = Op.getNumOperands();
14769 assert(Factor <= 8);
14770
14771 // i1 vectors need to be widened to i8
14772 if (VecVT.getVectorElementType() == MVT::i1)
14773 return widenVectorOpsToi8(N: Op, DL, DAG);
14774
14775 MVT ContainerVecVT = VecVT;
14776 if (VecVT.isFixedLengthVector())
14777 ContainerVecVT = getContainerForFixedLengthVector(VT: VecVT);
14778
14779 // If the VT is larger than LMUL=8, we need to split and reassemble.
14780 if ((ContainerVecVT.getSizeInBits().getKnownMinValue() * Factor) >
14781 (8 * RISCV::RVVBitsPerBlock) ||
14782 // In an unlikely case, a vector type might be legal to RISC-V but
14783 // exceeds the MVT limit, in which we also want to split it.
14784 (Subtarget.hasStdExtZvzip() && Factor == 2 &&
14785 !isTypeLegal(VT: VecVT.changeVectorElementCount(
14786 EC: VecVT.getVectorElementCount() * Factor)))) {
14787 SmallVector<SDValue, 8> Ops(Factor * 2);
14788 for (unsigned i = 0; i != Factor; ++i) {
14789 auto [OpLo, OpHi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: i);
14790 Ops[i] = OpLo;
14791 Ops[i + Factor] = OpHi;
14792 }
14793
14794 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
14795
14796 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
14797 Ops: ArrayRef(Ops).take_front(N: Factor)),
14798 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
14799 Ops: ArrayRef(Ops).drop_front(N: Factor))};
14800
14801 SmallVector<SDValue, 8> Concats(Factor);
14802 for (unsigned i = 0; i != Factor; ++i) {
14803 unsigned IdxLo = 2 * i;
14804 unsigned IdxHi = 2 * i + 1;
14805 Concats[i] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT,
14806 N1: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
14807 N2: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
14808 }
14809
14810 return DAG.getMergeValues(Ops: Concats, dl: DL);
14811 }
14812
14813 MVT XLenVT = Subtarget.getXLenVT();
14814 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: ContainerVecVT, DL, DAG, Subtarget);
14815
14816 SDValue Interleaved;
14817
14818 // Spill to the stack using a segment store for simplicity.
14819 if (Factor != 2) {
14820 EVT MemVT =
14821 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(),
14822 EC: VecVT.getVectorElementCount() * Factor);
14823
14824 // Allocate a stack slot.
14825 // Note that in the case where VecVT is fixed vector, even we later
14826 // create a container for each (fixed vector) operand, we still allocate
14827 // the stack with fixed vector size, rather than container size, as
14828 // fixed vector size is already sufficient.
14829 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
14830 SDValue StackPtr =
14831 DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
14832 EVT PtrVT = StackPtr.getValueType();
14833 auto &MF = DAG.getMachineFunction();
14834 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
14835 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
14836
14837 static const Intrinsic::ID IntrIds[] = {
14838 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
14839 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
14840 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
14841 Intrinsic::riscv_vsseg8_mask,
14842 };
14843
14844 unsigned Sz = Factor * ContainerVecVT.getVectorMinNumElements() *
14845 ContainerVecVT.getScalarSizeInBits();
14846 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: Factor);
14847
14848 SDValue StoredVal = DAG.getUNDEF(VT: VecTupTy);
14849 for (unsigned i = 0; i < Factor; i++) {
14850 SDValue OpVal = Op.getOperand(i);
14851 if (VecVT.isFixedLengthVector())
14852 OpVal = convertToScalableVector(VT: ContainerVecVT, V: OpVal, DAG, Subtarget);
14853 StoredVal = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: VecTupTy, N1: StoredVal,
14854 N2: OpVal, N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
14855 }
14856
14857 SDValue Ops[] = {DAG.getEntryNode(),
14858 DAG.getTargetConstant(Val: IntrIds[Factor - 2], DL, VT: XLenVT),
14859 StoredVal,
14860 StackPtr,
14861 Mask,
14862 VL,
14863 DAG.getTargetConstant(Val: Log2_64(Value: VecVT.getScalarSizeInBits()),
14864 DL, VT: XLenVT)};
14865
14866 SDValue Chain = DAG.getMemIntrinsicNode(
14867 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
14868 MemVT: VecVT.getVectorElementType(), PtrInfo, Alignment,
14869 Flags: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer());
14870
14871 SmallVector<SDValue, 8> Loads(Factor);
14872
14873 SDValue Increment = DAG.getTypeSize(DL, VT: PtrVT, TS: VecVT.getStoreSize());
14874 for (unsigned i = 0; i != Factor; ++i) {
14875 if (i != 0)
14876 StackPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: Increment);
14877
14878 Loads[i] = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
14879 }
14880
14881 return DAG.getMergeValues(Ops: Loads, dl: DL);
14882 }
14883
14884 if (Subtarget.hasStdExtZvzip() && !Op.getOperand(i: 0).isUndef() &&
14885 !Op.getOperand(i: 1).isUndef()) {
14886 MVT VT = Op->getSimpleValueType(ResNo: 0);
14887 MVT NewVT = VT.getDoubleNumVectorElementsVT();
14888 if (isLegalVTForZvzipInterleavedOperand(VT: NewVT, Subtarget)) {
14889 // Freeze the sources so we can increase their use count.
14890 SDValue V1 = DAG.getFreeze(V: Op->getOperand(Num: 0));
14891 SDValue V2 = DAG.getFreeze(V: Op->getOperand(Num: 1));
14892 SDValue Interleaved = lowerZvzipVZIP(Op0: V1, Op1: V2, DL, DAG, Subtarget);
14893 SDValue Lo = DAG.getExtractSubvector(DL, VT, Vec: Interleaved, Idx: 0);
14894 SDValue Hi = DAG.getExtractSubvector(DL, VT, Vec: Interleaved,
14895 Idx: VT.getVectorMinNumElements());
14896 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
14897 }
14898 }
14899
14900 // If the element type is smaller than ELEN, then we can interleave with
14901 // vwaddu.vv and vwmaccu.vx
14902 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
14903 Interleaved = getWideningInterleave(EvenV: Op.getOperand(i: 0), OddV: Op.getOperand(i: 1), DL,
14904 DAG, Subtarget);
14905 } else {
14906 // Otherwise, fallback to using vrgathere16.vv
14907 MVT ConcatVT = MVT::getVectorVT(VT: VecVT.getVectorElementType(),
14908 EC: VecVT.getVectorElementCount() * 2);
14909 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT,
14910 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
14911
14912 MVT IdxVT = ConcatVT.changeVectorElementType(EltVT: MVT::i16);
14913
14914 // 0 1 2 3 4 5 6 7 ...
14915 SDValue StepVec = DAG.getStepVector(DL, ResVT: IdxVT);
14916
14917 // 1 1 1 1 1 1 1 1 ...
14918 SDValue Ones = DAG.getSplatVector(VT: IdxVT, DL, Op: DAG.getConstant(Val: 1, DL, VT: XLenVT));
14919
14920 // 1 0 1 0 1 0 1 0 ...
14921 SDValue OddMask = DAG.getNode(Opcode: ISD::AND, DL, VT: IdxVT, N1: StepVec, N2: Ones);
14922 OddMask = DAG.getSetCC(
14923 DL, VT: IdxVT.changeVectorElementType(EltVT: MVT::i1), LHS: OddMask,
14924 RHS: DAG.getSplatVector(VT: IdxVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: XLenVT)),
14925 Cond: ISD::CondCode::SETNE);
14926
14927 SDValue VLMax = DAG.getSplatVector(VT: IdxVT, DL, Op: computeVLMax(VecVT, DL, DAG));
14928
14929 // Build up the index vector for interleaving the concatenated vector
14930 // 0 0 1 1 2 2 3 3 ...
14931 SDValue Idx = DAG.getNode(Opcode: ISD::SRL, DL, VT: IdxVT, N1: StepVec, N2: Ones);
14932 // 0 n 1 n+1 2 n+2 3 n+3 ...
14933 Idx =
14934 DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT: IdxVT, N1: Idx, N2: VLMax, N3: Idx, N4: OddMask, N5: VL);
14935
14936 // Then perform the interleave
14937 // v[0] v[n] v[1] v[n+1] v[2] v[n+2] v[3] v[n+3] ...
14938 SDValue TrueMask = getAllOnesMask(VecVT: IdxVT, VL, DL, DAG);
14939 Interleaved = DAG.getNode(Opcode: RISCVISD::VRGATHEREI16_VV_VL, DL, VT: ConcatVT,
14940 N1: Concat, N2: Idx, N3: DAG.getUNDEF(VT: ConcatVT), N4: TrueMask, N5: VL);
14941 }
14942
14943 // Extract the two halves from the interleaved result
14944 SDValue Lo = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved, Idx: 0);
14945 SDValue Hi = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved,
14946 Idx: VecVT.getVectorMinNumElements());
14947
14948 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
14949}
14950
14951// Lower step_vector to the vid instruction. Any non-identity step value must
14952// be accounted for my manual expansion.
14953SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
14954 SelectionDAG &DAG) const {
14955 SDLoc DL(Op);
14956 MVT VT = Op.getSimpleValueType();
14957 assert(VT.isScalableVector() && "Expected scalable vector");
14958 MVT XLenVT = Subtarget.getXLenVT();
14959 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
14960 SDValue StepVec = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT, N1: Mask, N2: VL);
14961 uint64_t StepValImm = Op.getConstantOperandVal(i: 0);
14962 if (StepValImm != 1) {
14963 if (isPowerOf2_64(Value: StepValImm)) {
14964 SDValue StepVal =
14965 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
14966 N2: DAG.getConstant(Val: Log2_64(Value: StepValImm), DL, VT: XLenVT), N3: VL);
14967 StepVec = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: StepVec, N2: StepVal);
14968 } else {
14969 SDValue StepVal = lowerScalarSplat(
14970 Passthru: SDValue(), Scalar: DAG.getConstant(Val: StepValImm, DL, VT: VT.getVectorElementType()),
14971 VL, VT, DL, DAG, Subtarget);
14972 StepVec = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: StepVec, N2: StepVal);
14973 }
14974 }
14975 return StepVec;
14976}
14977
14978// Implement vector_reverse using vrgather.vv with indices determined by
14979// subtracting the id of each element from (VLMAX-1). This will convert
14980// the indices like so:
14981// (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
14982// TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
14983SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
14984 SelectionDAG &DAG) const {
14985 SDLoc DL(Op);
14986 MVT VecVT = Op.getSimpleValueType();
14987
14988 // Reverse a 64-bit packed vector on RV32 by reversing each 32-bit half and
14989 // swapping them.
14990 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
14991 assert(!Subtarget.is64Bit() && VecVT.getSizeInBits() == 64 &&
14992 "Unexpected packed VECTOR_REVERSE type");
14993 SDValue V = Op.getOperand(i: 0);
14994 if (VecVT == MVT::v2i32) {
14995 // A 2-element reverse is just an element swap.
14996 SDValue Lo = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx: 0);
14997 SDValue Hi = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx: 1);
14998 return DAG.getBuildVector(VT: VecVT, DL, Ops: {Hi, Lo});
14999 }
15000 auto [Lo, Hi] = DAG.SplitVector(N: V, DL);
15001 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Lo.getSimpleValueType(), Operand: Lo);
15002 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Hi.getSimpleValueType(), Operand: Hi);
15003 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Hi, N2: Lo);
15004 }
15005
15006 if (VecVT.getVectorElementType() == MVT::i1) {
15007 MVT WidenVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
15008 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: Op.getOperand(i: 0));
15009 SDValue Op2 = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: WidenVT, Operand: Op1);
15010 return DAG.getSetCC(DL, VT: VecVT, LHS: Op2,
15011 RHS: DAG.getConstant(Val: 0, DL, VT: Op2.getValueType()), Cond: ISD::SETNE);
15012 }
15013
15014 MVT ContainerVT = VecVT;
15015 SDValue Vec = Op.getOperand(i: 0);
15016 if (VecVT.isFixedLengthVector()) {
15017 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
15018 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
15019 }
15020
15021 MVT XLenVT = Subtarget.getXLenVT();
15022 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
15023
15024 // On some uarchs vrgather.vv will read from every input register for each
15025 // output register, regardless of the indices. However to reverse a vector
15026 // each output register only needs to read from one register. So decompose it
15027 // into LMUL * M1 vrgather.vvs, so we get O(LMUL) performance instead of
15028 // O(LMUL^2).
15029 //
15030 // vsetvli a1, zero, e64, m4, ta, ma
15031 // vrgatherei16.vv v12, v8, v16
15032 // ->
15033 // vsetvli a1, zero, e64, m1, ta, ma
15034 // vrgather.vv v15, v8, v16
15035 // vrgather.vv v14, v9, v16
15036 // vrgather.vv v13, v10, v16
15037 // vrgather.vv v12, v11, v16
15038 if (ContainerVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT)) &&
15039 ContainerVT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
15040 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
15041 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Lo.getValueType(), Operand: Lo);
15042 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Hi.getValueType(), Operand: Hi);
15043 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ContainerVT, N1: Hi, N2: Lo);
15044
15045 // Fixed length vectors might not fit exactly into their container, and so
15046 // leave a gap in the front of the vector after being reversed. Slide this
15047 // away.
15048 //
15049 // x x x x 3 2 1 0 <- v4i16 @ vlen=128
15050 // 0 1 2 3 x x x x <- reverse
15051 // x x x x 0 1 2 3 <- vslidedown.vx
15052 if (VecVT.isFixedLengthVector()) {
15053 SDValue Offset = DAG.getNode(
15054 Opcode: ISD::SUB, DL, VT: XLenVT,
15055 N1: DAG.getElementCount(DL, VT: XLenVT, EC: ContainerVT.getVectorElementCount()),
15056 N2: DAG.getElementCount(DL, VT: XLenVT, EC: VecVT.getVectorElementCount()));
15057 Concat =
15058 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
15059 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Concat, Offset, Mask, VL);
15060 Concat = convertFromScalableVector(VT: VecVT, V: Concat, DAG, Subtarget);
15061 }
15062 return Concat;
15063 }
15064
15065 unsigned EltSize = ContainerVT.getScalarSizeInBits();
15066 unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
15067 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
15068 unsigned MaxVLMAX =
15069 VecVT.isFixedLengthVector()
15070 ? VecVT.getVectorNumElements()
15071 : RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
15072
15073 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
15074 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
15075
15076 // If this is SEW=8 and VLMAX is potentially more than 256, we need
15077 // to use vrgatherei16.vv.
15078 if (MaxVLMAX > 256 && EltSize == 8) {
15079 // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
15080 // Reverse each half, then reassemble them in reverse order.
15081 // NOTE: It's also possible that after splitting that VLMAX no longer
15082 // requires vrgatherei16.vv.
15083 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
15084 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
15085 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
15086 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: LoVT, Operand: Lo);
15087 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: HiVT, Operand: Hi);
15088 // Reassemble the low and high pieces reversed.
15089 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Hi, N2: Lo);
15090 }
15091
15092 // Just promote the int type to i16 which will double the LMUL.
15093 IntVT = MVT::getVectorVT(VT: MVT::i16, EC: ContainerVT.getVectorElementCount());
15094 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
15095 }
15096
15097 // At LMUL > 1, do the index computation in 16 bits to reduce register
15098 // pressure.
15099 if (IntVT.getScalarType().bitsGT(VT: MVT::i16) &&
15100 IntVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: IntVT))) {
15101 assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
15102 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
15103 IntVT = IntVT.changeVectorElementType(EltVT: MVT::i16);
15104 }
15105
15106 // Calculate VLMAX-1 for the desired SEW.
15107 SDValue VLMinus1 = DAG.getNode(
15108 Opcode: ISD::SUB, DL, VT: XLenVT,
15109 N1: DAG.getElementCount(DL, VT: XLenVT, EC: VecVT.getVectorElementCount()),
15110 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
15111
15112 // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
15113 bool IsRV32E64 =
15114 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
15115 SDValue SplatVL;
15116 if (!IsRV32E64)
15117 SplatVL = DAG.getSplatVector(VT: IntVT, DL, Op: VLMinus1);
15118 else
15119 SplatVL = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IntVT, N1: DAG.getUNDEF(VT: IntVT),
15120 N2: VLMinus1, N3: DAG.getRegister(Reg: RISCV::X0, VT: XLenVT));
15121
15122 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: IntVT, N1: Mask, N2: VL);
15123 SDValue Indices = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: IntVT, N1: SplatVL, N2: VID,
15124 N3: DAG.getUNDEF(VT: IntVT), N4: Mask, N5: VL);
15125
15126 SDValue Gather = DAG.getNode(Opcode: GatherOpc, DL, VT: ContainerVT, N1: Vec, N2: Indices,
15127 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
15128 if (VecVT.isFixedLengthVector())
15129 Gather = convertFromScalableVector(VT: VecVT, V: Gather, DAG, Subtarget);
15130 return Gather;
15131}
15132
15133SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
15134 SelectionDAG &DAG) const {
15135 SDLoc DL(Op);
15136 SDValue V1 = Op.getOperand(i: 0);
15137 SDValue V2 = Op.getOperand(i: 1);
15138 SDValue Offset = Op.getOperand(i: 2);
15139 MVT XLenVT = Subtarget.getXLenVT();
15140 MVT VecVT = Op.getSimpleValueType();
15141
15142 SDValue VLMax = computeVLMax(VecVT, DL, DAG);
15143
15144 SDValue DownOffset, UpOffset;
15145 if (Op.getOpcode() == ISD::VECTOR_SPLICE_LEFT) {
15146 // The operand is a TargetConstant, we need to rebuild it as a regular
15147 // constant.
15148 DownOffset = Offset;
15149 UpOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: Offset);
15150 } else {
15151 // The operand is a TargetConstant, we need to rebuild it as a regular
15152 // constant rather than negating the original operand.
15153 UpOffset = Offset;
15154 DownOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: Offset);
15155 }
15156
15157 SDValue TrueMask = getAllOnesMask(VecVT, VL: VLMax, DL, DAG);
15158
15159 SDValue SlideDown = getVSlidedown(
15160 DAG, Subtarget, DL, VT: VecVT, Passthru: DAG.getUNDEF(VT: VecVT), Op: V1, Offset: DownOffset, Mask: TrueMask,
15161 VL: Subtarget.hasVLDependentLatency() ? UpOffset
15162 : DAG.getRegister(Reg: RISCV::X0, VT: XLenVT));
15163 return getVSlideup(DAG, Subtarget, DL, VT: VecVT, Passthru: SlideDown, Op: V2, Offset: UpOffset,
15164 Mask: TrueMask, VL: DAG.getRegister(Reg: RISCV::X0, VT: XLenVT),
15165 Policy: RISCVVType::TAIL_AGNOSTIC);
15166}
15167
15168SDValue
15169RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
15170 SelectionDAG &DAG) const {
15171 SDLoc DL(Op);
15172 auto *Load = cast<LoadSDNode>(Val&: Op);
15173
15174 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
15175 Load->getMemoryVT(),
15176 *Load->getMemOperand()) &&
15177 "Expecting a correctly-aligned load");
15178
15179 MVT VT = Op.getSimpleValueType();
15180 MVT XLenVT = Subtarget.getXLenVT();
15181 MVT ContainerVT = getContainerForFixedLengthVector(VT);
15182
15183 // If we know the exact VLEN and our fixed length vector completely fills
15184 // the container, use a whole register load instead.
15185 const auto [MinVLMAX, MaxVLMAX] =
15186 RISCVTargetLowering::computeVLMAXBounds(VecVT: ContainerVT, Subtarget);
15187 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
15188 RISCVTargetLowering::getM1VT(VT: ContainerVT).bitsLE(VT: ContainerVT)) {
15189 MachineMemOperand *MMO = Load->getMemOperand();
15190 SDValue NewLoad =
15191 DAG.getLoad(VT: ContainerVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
15192 PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getBaseAlign(), MMOFlags: MMO->getFlags(),
15193 Metadata: MMOMetadata(MMO->getAAInfo(), MMO->getRanges()));
15194 SDValue Result = convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
15195 return DAG.getMergeValues(Ops: {Result, NewLoad.getValue(R: 1)}, dl: DL);
15196 }
15197
15198 SDValue VL = DAG.getConstant(Val: VT.getVectorNumElements(), DL, VT: XLenVT);
15199
15200 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
15201 SDValue IntID = DAG.getTargetConstant(
15202 Val: IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, VT: XLenVT);
15203 SmallVector<SDValue, 4> Ops{Load->getChain(), IntID};
15204 if (!IsMaskOp)
15205 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
15206 Ops.push_back(Elt: Load->getBasePtr());
15207 Ops.push_back(Elt: VL);
15208 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
15209 SDValue NewLoad =
15210 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
15211 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
15212
15213 SDValue Result = convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
15214 return DAG.getMergeValues(Ops: {Result, NewLoad.getValue(R: 1)}, dl: DL);
15215}
15216
15217SDValue
15218RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
15219 SelectionDAG &DAG) const {
15220 SDLoc DL(Op);
15221 auto *Store = cast<StoreSDNode>(Val&: Op);
15222
15223 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
15224 Store->getMemoryVT(),
15225 *Store->getMemOperand()) &&
15226 "Expecting a correctly-aligned store");
15227
15228 SDValue StoreVal = Store->getValue();
15229 MVT VT = StoreVal.getSimpleValueType();
15230 MVT XLenVT = Subtarget.getXLenVT();
15231
15232 // If the size less than a byte, we need to pad with zeros to make a byte.
15233 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
15234 VT = MVT::v8i1;
15235 StoreVal =
15236 DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT), SubVec: StoreVal, Idx: 0);
15237 }
15238
15239 MVT ContainerVT = getContainerForFixedLengthVector(VT);
15240
15241 SDValue NewValue =
15242 convertToScalableVector(VT: ContainerVT, V: StoreVal, DAG, Subtarget);
15243
15244 // If we know the exact VLEN and our fixed length vector completely fills
15245 // the container, use a whole register store instead.
15246 const auto [MinVLMAX, MaxVLMAX] =
15247 RISCVTargetLowering::computeVLMAXBounds(VecVT: ContainerVT, Subtarget);
15248 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
15249 RISCVTargetLowering::getM1VT(VT: ContainerVT).bitsLE(VT: ContainerVT)) {
15250 MachineMemOperand *MMO = Store->getMemOperand();
15251 return DAG.getStore(Chain: Store->getChain(), dl: DL, Val: NewValue, Ptr: Store->getBasePtr(),
15252 PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getBaseAlign(),
15253 MMOFlags: MMO->getFlags(), Metadata: MMO->getAAInfo());
15254 }
15255
15256 SDValue VL = DAG.getConstant(Val: VT.getVectorNumElements(), DL, VT: XLenVT);
15257
15258 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
15259 SDValue IntID = DAG.getTargetConstant(
15260 Val: IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, VT: XLenVT);
15261 return DAG.getMemIntrinsicNode(
15262 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
15263 Ops: {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
15264 MemVT: Store->getMemoryVT(), MMO: Store->getMemOperand());
15265}
15266
15267SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
15268 SelectionDAG &DAG) const {
15269 SDLoc DL(Op);
15270 MVT VT = Op.getSimpleValueType();
15271
15272 const auto *MemSD = cast<MemSDNode>(Val&: Op);
15273 EVT MemVT = MemSD->getMemoryVT();
15274 MachineMemOperand *MMO = MemSD->getMemOperand();
15275 SDValue Chain = MemSD->getChain();
15276 SDValue BasePtr = MemSD->getBasePtr();
15277
15278 SDValue Mask, PassThru, VL;
15279 bool IsExpandingLoad = false;
15280 if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Val&: Op)) {
15281 Mask = VPLoad->getMask();
15282 PassThru = DAG.getUNDEF(VT);
15283 VL = VPLoad->getVectorLength();
15284 } else {
15285 const auto *MLoad = cast<MaskedLoadSDNode>(Val&: Op);
15286 Mask = MLoad->getMask();
15287 PassThru = MLoad->getPassThru();
15288 IsExpandingLoad = MLoad->isExpandingLoad();
15289 }
15290
15291 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
15292
15293 MVT XLenVT = Subtarget.getXLenVT();
15294
15295 MVT ContainerVT = VT;
15296 if (VT.isFixedLengthVector()) {
15297 ContainerVT = getContainerForFixedLengthVector(VT);
15298 PassThru = convertToScalableVector(VT: ContainerVT, V: PassThru, DAG, Subtarget);
15299 if (!IsUnmasked) {
15300 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15301 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15302 }
15303 }
15304
15305 if (!VL)
15306 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
15307
15308 SDValue ExpandingVL;
15309 if (!IsUnmasked && IsExpandingLoad) {
15310 ExpandingVL = VL;
15311 VL =
15312 DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Mask,
15313 N2: getAllOnesMask(VecVT: Mask.getSimpleValueType(), VL, DL, DAG), N3: VL);
15314 }
15315
15316 unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
15317 : Intrinsic::riscv_vle_mask;
15318 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
15319 if (IntID == Intrinsic::riscv_vle)
15320 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
15321 else
15322 Ops.push_back(Elt: PassThru);
15323 Ops.push_back(Elt: BasePtr);
15324 if (IntID == Intrinsic::riscv_vle_mask)
15325 Ops.push_back(Elt: Mask);
15326 Ops.push_back(Elt: VL);
15327 if (IntID == Intrinsic::riscv_vle_mask)
15328 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT));
15329
15330 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
15331
15332 SDValue Result =
15333 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
15334 Chain = Result.getValue(R: 1);
15335 if (ExpandingVL) {
15336 MVT IndexVT = ContainerVT;
15337 if (ContainerVT.isFloatingPoint())
15338 IndexVT = ContainerVT.changeVectorElementTypeToInteger();
15339
15340 MVT IndexEltVT = IndexVT.getVectorElementType();
15341 bool UseVRGATHEREI16 = false;
15342 // If index vector is an i8 vector and the element count exceeds 256, we
15343 // should change the element type of index vector to i16 to avoid
15344 // overflow.
15345 if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
15346 // FIXME: We need to do vector splitting manually for LMUL=8 cases.
15347 assert(getLMUL(IndexVT) != RISCVVType::LMUL_8);
15348 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
15349 UseVRGATHEREI16 = true;
15350 }
15351
15352 SDValue Iota =
15353 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: IndexVT,
15354 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_viota, DL, VT: XLenVT),
15355 N2: DAG.getUNDEF(VT: IndexVT), N3: Mask, N4: ExpandingVL);
15356 Result =
15357 DAG.getNode(Opcode: UseVRGATHEREI16 ? RISCVISD::VRGATHEREI16_VV_VL
15358 : RISCVISD::VRGATHER_VV_VL,
15359 DL, VT: ContainerVT, N1: Result, N2: Iota, N3: PassThru, N4: Mask, N5: ExpandingVL);
15360 }
15361
15362 if (VT.isFixedLengthVector())
15363 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15364
15365 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
15366}
15367
15368SDValue RISCVTargetLowering::lowerLoadFF(SDValue Op, SelectionDAG &DAG) const {
15369 SDLoc DL(Op);
15370 MVT VT = Op->getSimpleValueType(ResNo: 0);
15371
15372 const auto *VPLoadFF = cast<VPLoadFFSDNode>(Val&: Op);
15373 EVT MemVT = VPLoadFF->getMemoryVT();
15374 MachineMemOperand *MMO = VPLoadFF->getMemOperand();
15375 SDValue Chain = VPLoadFF->getChain();
15376 SDValue BasePtr = VPLoadFF->getBasePtr();
15377
15378 SDValue Mask = VPLoadFF->getMask();
15379 SDValue VL = VPLoadFF->getVectorLength();
15380
15381 MVT XLenVT = Subtarget.getXLenVT();
15382
15383 MVT ContainerVT = VT;
15384 if (VT.isFixedLengthVector()) {
15385 ContainerVT = getContainerForFixedLengthVector(VT);
15386 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15387 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15388 }
15389
15390 unsigned IntID = Intrinsic::riscv_vleff_mask;
15391 SDValue Ops[] = {
15392 Chain,
15393 DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT),
15394 DAG.getUNDEF(VT: ContainerVT),
15395 BasePtr,
15396 Mask,
15397 VL,
15398 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT)};
15399
15400 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, Op->getValueType(ResNo: 1), MVT::Other});
15401
15402 SDValue Result =
15403 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
15404 SDValue OutVL = Result.getValue(R: 1);
15405 Chain = Result.getValue(R: 2);
15406
15407 if (VT.isFixedLengthVector())
15408 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15409
15410 return DAG.getMergeValues(Ops: {Result, OutVL, Chain}, dl: DL);
15411}
15412
15413SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
15414 SelectionDAG &DAG) const {
15415 SDLoc DL(Op);
15416
15417 const auto *MemSD = cast<MemSDNode>(Val&: Op);
15418 EVT MemVT = MemSD->getMemoryVT();
15419 MachineMemOperand *MMO = MemSD->getMemOperand();
15420 SDValue Chain = MemSD->getChain();
15421 SDValue BasePtr = MemSD->getBasePtr();
15422 SDValue Val, Mask, VL;
15423
15424 bool IsCompressingStore = false;
15425 if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Val&: Op)) {
15426 Val = VPStore->getValue();
15427 Mask = VPStore->getMask();
15428 VL = VPStore->getVectorLength();
15429 } else {
15430 const auto *MStore = cast<MaskedStoreSDNode>(Val&: Op);
15431 Val = MStore->getValue();
15432 Mask = MStore->getMask();
15433 IsCompressingStore = MStore->isCompressingStore();
15434 }
15435
15436 bool IsUnmasked =
15437 ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()) || IsCompressingStore;
15438
15439 MVT VT = Val.getSimpleValueType();
15440 MVT XLenVT = Subtarget.getXLenVT();
15441
15442 MVT ContainerVT = VT;
15443 if (VT.isFixedLengthVector()) {
15444 ContainerVT = getContainerForFixedLengthVector(VT);
15445
15446 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
15447 if (!IsUnmasked || IsCompressingStore) {
15448 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15449 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15450 }
15451 }
15452
15453 if (!VL)
15454 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
15455
15456 if (IsCompressingStore) {
15457 Val = DAG.getNode(
15458 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ContainerVT,
15459 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_vcompress, DL, VT: XLenVT),
15460 N2: DAG.getUNDEF(VT: ContainerVT), N3: Val, N4: Mask, N5: VL);
15461 VL =
15462 DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Mask,
15463 N2: getAllOnesMask(VecVT: Mask.getSimpleValueType(), VL, DL, DAG), N3: VL);
15464 }
15465
15466 unsigned IntID =
15467 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
15468 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
15469 Ops.push_back(Elt: Val);
15470 Ops.push_back(Elt: BasePtr);
15471 if (!IsUnmasked)
15472 Ops.push_back(Elt: Mask);
15473 Ops.push_back(Elt: VL);
15474
15475 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
15476 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
15477}
15478
15479SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op,
15480 SelectionDAG &DAG) const {
15481 SDLoc DL(Op);
15482 SDValue Val = Op.getOperand(i: 0);
15483 SDValue Mask = Op.getOperand(i: 1);
15484 SDValue Passthru = Op.getOperand(i: 2);
15485
15486 MVT VT = Val.getSimpleValueType();
15487 MVT XLenVT = Subtarget.getXLenVT();
15488 MVT ContainerVT = VT;
15489 if (VT.isFixedLengthVector()) {
15490 ContainerVT = getContainerForFixedLengthVector(VT);
15491 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15492 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
15493 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15494 Passthru = convertToScalableVector(VT: ContainerVT, V: Passthru, DAG, Subtarget);
15495 }
15496
15497 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
15498 SDValue Res =
15499 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ContainerVT,
15500 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_vcompress, DL, VT: XLenVT),
15501 N2: Passthru, N3: Val, N4: Mask, N5: VL);
15502
15503 if (VT.isFixedLengthVector())
15504 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
15505
15506 return Res;
15507}
15508
15509SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op,
15510 SelectionDAG &DAG) const {
15511 unsigned Opc = Op.getOpcode();
15512 SDLoc DL(Op);
15513 SDValue Chain = Op.getOperand(i: 0);
15514 SDValue Op1 = Op.getOperand(i: 1);
15515 SDValue Op2 = Op.getOperand(i: 2);
15516 SDValue CC = Op.getOperand(i: 3);
15517 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
15518 MVT VT = Op.getSimpleValueType();
15519 MVT InVT = Op1.getSimpleValueType();
15520
15521 // RVV VMFEQ/VMFNE ignores qNan, so we expand strict_fsetccs with OEQ/UNE
15522 // condition code.
15523 if (Opc == ISD::STRICT_FSETCCS) {
15524 // Expand strict_fsetccs(x, oeq) to
15525 // (and strict_fsetccs(x, y, oge), strict_fsetccs(x, y, ole))
15526 SDVTList VTList = Op->getVTList();
15527 if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) {
15528 SDValue OLECCVal = DAG.getCondCode(Cond: ISD::SETOLE);
15529 SDValue Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op1,
15530 N3: Op2, N4: OLECCVal);
15531 SDValue Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op2,
15532 N3: Op1, N4: OLECCVal);
15533 SDValue OutChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
15534 N1: Tmp1.getValue(R: 1), N2: Tmp2.getValue(R: 1));
15535 // Tmp1 and Tmp2 might be the same node.
15536 if (Tmp1 != Tmp2)
15537 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp1, N2: Tmp2);
15538 return DAG.getMergeValues(Ops: {Tmp1, OutChain}, dl: DL);
15539 }
15540
15541 // Expand (strict_fsetccs x, y, une) to (not (strict_fsetccs x, y, oeq))
15542 if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) {
15543 SDValue OEQCCVal = DAG.getCondCode(Cond: ISD::SETOEQ);
15544 SDValue OEQ = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op1,
15545 N3: Op2, N4: OEQCCVal);
15546 SDValue Res = DAG.getNOT(DL, Val: OEQ, VT);
15547 return DAG.getMergeValues(Ops: {Res, OEQ.getValue(R: 1)}, dl: DL);
15548 }
15549 }
15550
15551 MVT ContainerInVT = InVT;
15552 if (InVT.isFixedLengthVector()) {
15553 ContainerInVT = getContainerForFixedLengthVector(VT: InVT);
15554 Op1 = convertToScalableVector(VT: ContainerInVT, V: Op1, DAG, Subtarget);
15555 Op2 = convertToScalableVector(VT: ContainerInVT, V: Op2, DAG, Subtarget);
15556 }
15557 MVT MaskVT = getMaskTypeFor(VecVT: ContainerInVT);
15558
15559 auto [Mask, VL] = getDefaultVLOps(VecVT: InVT, ContainerVT: ContainerInVT, DL, DAG, Subtarget);
15560
15561 SDValue Res;
15562 if (Opc == ISD::STRICT_FSETCC &&
15563 (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE ||
15564 CCVal == ISD::SETOLE)) {
15565 // VMFLT/VMFLE/VMFGT/VMFGE raise exception for qNan. Generate a mask to only
15566 // active when both input elements are ordered.
15567 SDValue True = getAllOnesMask(VecVT: ContainerInVT, VL, DL, DAG);
15568 SDValue OrderMask1 = DAG.getNode(
15569 Opcode: RISCVISD::STRICT_FSETCC_VL, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
15570 Ops: {Chain, Op1, Op1, DAG.getCondCode(Cond: ISD::SETOEQ), DAG.getUNDEF(VT: MaskVT),
15571 True, VL});
15572 SDValue OrderMask2 = DAG.getNode(
15573 Opcode: RISCVISD::STRICT_FSETCC_VL, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
15574 Ops: {Chain, Op2, Op2, DAG.getCondCode(Cond: ISD::SETOEQ), DAG.getUNDEF(VT: MaskVT),
15575 True, VL});
15576 Mask =
15577 DAG.getNode(Opcode: RISCVISD::VMAND_VL, DL, VT: MaskVT, N1: OrderMask1, N2: OrderMask2, N3: VL);
15578 // Use Mask as the passthru operand to let the result be 0 if either of the
15579 // inputs is unordered.
15580 Res = DAG.getNode(Opcode: RISCVISD::STRICT_FSETCCS_VL, DL,
15581 VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
15582 Ops: {Chain, Op1, Op2, CC, Mask, Mask, VL});
15583 } else {
15584 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
15585 : RISCVISD::STRICT_FSETCCS_VL;
15586 Res = DAG.getNode(Opcode: RVVOpc, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
15587 Ops: {Chain, Op1, Op2, CC, DAG.getUNDEF(VT: MaskVT), Mask, VL});
15588 }
15589
15590 if (VT.isFixedLengthVector()) {
15591 SDValue SubVec = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
15592 return DAG.getMergeValues(Ops: {SubVec, Res.getValue(R: 1)}, dl: DL);
15593 }
15594 return Res;
15595}
15596
15597// Lower vector ABS to smax(X, sub(0, X)).
15598SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
15599 SDLoc DL(Op);
15600 MVT VT = Op.getSimpleValueType();
15601 SDValue X = Op.getOperand(i: 0);
15602
15603 assert(VT.isFixedLengthVector() && "Unexpected type for ISD::ABS");
15604
15605 MVT ContainerVT = getContainerForFixedLengthVector(VT);
15606 X = convertToScalableVector(VT: ContainerVT, V: X, DAG, Subtarget);
15607
15608 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
15609
15610 SDValue SplatZero = DAG.getNode(
15611 Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT),
15612 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()), N3: VL);
15613 SDValue Result;
15614 if (Subtarget.hasStdExtZvabd()) {
15615 Result = DAG.getNode(Opcode: RISCVISD::ABDS_VL, DL, VT: ContainerVT, N1: X, N2: SplatZero,
15616 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
15617 } else {
15618 SDValue NegX = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: ContainerVT, N1: SplatZero, N2: X,
15619 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
15620 Result = DAG.getNode(Opcode: RISCVISD::SMAX_VL, DL, VT: ContainerVT, N1: X, N2: NegX,
15621 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
15622 }
15623 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15624}
15625
15626SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op,
15627 SelectionDAG &DAG) const {
15628 const auto &TSInfo =
15629 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
15630
15631 unsigned NewOpc = getRISCVVLOp(Op);
15632 bool HasPassthruOp = TSInfo.hasPassthruOp(Opcode: NewOpc);
15633 bool HasMask = TSInfo.hasMaskOp(Opcode: NewOpc);
15634
15635 MVT VT = Op.getSimpleValueType();
15636 MVT ContainerVT = getContainerForFixedLengthVector(VT);
15637
15638 // Create list of operands by converting existing ones to scalable types.
15639 SmallVector<SDValue, 6> Ops;
15640 for (const SDValue &V : Op->op_values()) {
15641 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
15642
15643 // Pass through non-vector operands.
15644 if (!V.getValueType().isVector()) {
15645 Ops.push_back(Elt: V);
15646 continue;
15647 }
15648
15649 // "cast" fixed length vector to a scalable vector.
15650 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
15651 "Only fixed length vectors are supported!");
15652 MVT VContainerVT = ContainerVT.changeVectorElementType(
15653 EltVT: V.getSimpleValueType().getVectorElementType());
15654 Ops.push_back(Elt: convertToScalableVector(VT: VContainerVT, V, DAG, Subtarget));
15655 }
15656
15657 SDLoc DL(Op);
15658 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
15659 if (HasPassthruOp)
15660 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
15661 if (HasMask)
15662 Ops.push_back(Elt: Mask);
15663 Ops.push_back(Elt: VL);
15664
15665 // StrictFP operations have two result values. Their lowered result should
15666 // have same result count.
15667 if (Op->isStrictFPOpcode()) {
15668 SDValue ScalableRes =
15669 DAG.getNode(Opcode: NewOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), Ops,
15670 Flags: Op->getFlags());
15671 SDValue SubVec = convertFromScalableVector(VT, V: ScalableRes, DAG, Subtarget);
15672 return DAG.getMergeValues(Ops: {SubVec, ScalableRes.getValue(R: 1)}, dl: DL);
15673 }
15674
15675 SDValue ScalableRes =
15676 DAG.getNode(Opcode: NewOpc, DL, VT: ContainerVT, Ops, Flags: Op->getFlags());
15677 return convertFromScalableVector(VT, V: ScalableRes, DAG, Subtarget);
15678}
15679
15680// Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
15681// * Operands of each node are assumed to be in the same order.
15682// * The EVL operand is promoted from i32 to i64 on RV64.
15683// * Fixed-length vectors are converted to their scalable-vector container
15684// types.
15685SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
15686 const auto &TSInfo =
15687 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
15688
15689 unsigned RISCVISDOpc = getRISCVVLOp(Op);
15690 bool HasPassthruOp = TSInfo.hasPassthruOp(Opcode: RISCVISDOpc);
15691
15692 SDLoc DL(Op);
15693 MVT VT = Op.getSimpleValueType();
15694 SmallVector<SDValue, 4> Ops;
15695
15696 MVT ContainerVT = VT;
15697 if (VT.isFixedLengthVector())
15698 ContainerVT = getContainerForFixedLengthVector(VT);
15699
15700 for (const auto &OpIdx : enumerate(First: Op->ops())) {
15701 SDValue V = OpIdx.value();
15702 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
15703 // Add dummy passthru value before the mask. Or if there isn't a mask,
15704 // before EVL.
15705 if (HasPassthruOp) {
15706 auto MaskIdx = ISD::getVPMaskIdx(Opcode: Op.getOpcode());
15707 if (MaskIdx) {
15708 if (*MaskIdx == OpIdx.index())
15709 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
15710 } else if (ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) ==
15711 OpIdx.index()) {
15712 assert(Op.getOpcode() == ISD::VP_MERGE);
15713 // For VP_MERGE, copy the false operand instead of an undef value.
15714 Ops.push_back(Elt: Ops.back());
15715 }
15716 }
15717 // VFCVT_RM_X_F_VL requires a rounding mode to be injected before the VL.
15718 if (RISCVISDOpc == RISCVISD::VFCVT_RM_X_F_VL &&
15719 ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) == OpIdx.index())
15720 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVFPRndMode::DYN, DL,
15721 VT: Subtarget.getXLenVT()));
15722 // Pass through operands which aren't fixed-length vectors.
15723 if (!V.getValueType().isFixedLengthVector()) {
15724 Ops.push_back(Elt: V);
15725 continue;
15726 }
15727 // "cast" fixed length vector to a scalable vector.
15728 MVT OpVT = V.getSimpleValueType();
15729 MVT ContainerVT = getContainerForFixedLengthVector(VT: OpVT);
15730 assert(useRVVForFixedLengthVectorVT(OpVT) &&
15731 "Only fixed length vectors are supported!");
15732 Ops.push_back(Elt: convertToScalableVector(VT: ContainerVT, V, DAG, Subtarget));
15733 }
15734
15735 if (!VT.isFixedLengthVector())
15736 return DAG.getNode(Opcode: RISCVISDOpc, DL, VT, Ops, Flags: Op->getFlags());
15737
15738 SDValue VPOp = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: ContainerVT, Ops, Flags: Op->getFlags());
15739
15740 return convertFromScalableVector(VT, V: VPOp, DAG, Subtarget);
15741}
15742
15743SDValue RISCVTargetLowering::lowerVPMergeMask(SDValue Op,
15744 SelectionDAG &DAG) const {
15745 SDLoc DL(Op);
15746 MVT VT = Op.getSimpleValueType();
15747 MVT XLenVT = Subtarget.getXLenVT();
15748
15749 SDValue Mask = Op.getOperand(i: 0);
15750 SDValue TrueVal = Op.getOperand(i: 1);
15751 SDValue FalseVal = Op.getOperand(i: 2);
15752 SDValue VL = Op.getOperand(i: 3);
15753
15754 // Use default legalization if a vector of EVL type would be legal.
15755 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VL.getValueType(),
15756 EC: VT.getVectorElementCount());
15757 if (isTypeLegal(VT: EVLVecVT))
15758 return SDValue();
15759
15760 MVT ContainerVT = VT;
15761 if (VT.isFixedLengthVector()) {
15762 ContainerVT = getContainerForFixedLengthVector(VT);
15763 Mask = convertToScalableVector(VT: ContainerVT, V: Mask, DAG, Subtarget);
15764 TrueVal = convertToScalableVector(VT: ContainerVT, V: TrueVal, DAG, Subtarget);
15765 FalseVal = convertToScalableVector(VT: ContainerVT, V: FalseVal, DAG, Subtarget);
15766 }
15767
15768 // Promote to a vector of i8.
15769 MVT PromotedVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
15770
15771 // Promote TrueVal and FalseVal using VLMax.
15772 // FIXME: Is there a better way to do this?
15773 SDValue VLMax = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
15774 SDValue SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: PromotedVT,
15775 N1: DAG.getUNDEF(VT: PromotedVT),
15776 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: VLMax);
15777 SDValue SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: PromotedVT,
15778 N1: DAG.getUNDEF(VT: PromotedVT),
15779 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: VLMax);
15780 TrueVal = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: TrueVal, N2: SplatOne,
15781 N3: SplatZero, N4: DAG.getUNDEF(VT: PromotedVT), N5: VL);
15782 // Any element past VL uses FalseVal, so use VLMax
15783 FalseVal = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: FalseVal,
15784 N2: SplatOne, N3: SplatZero, N4: DAG.getUNDEF(VT: PromotedVT), N5: VLMax);
15785
15786 // VP_MERGE the two promoted values.
15787 SDValue VPMerge = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: Mask,
15788 N2: TrueVal, N3: FalseVal, N4: FalseVal, N5: VL);
15789
15790 // Convert back to mask.
15791 SDValue TrueMask = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
15792 SDValue Result = DAG.getNode(
15793 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
15794 Ops: {VPMerge, DAG.getConstant(Val: 0, DL, VT: PromotedVT), DAG.getCondCode(Cond: ISD::SETNE),
15795 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), TrueMask, VLMax});
15796
15797 if (VT.isFixedLengthVector())
15798 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15799 return Result;
15800}
15801
15802SDValue
15803RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
15804 SelectionDAG &DAG) const {
15805 using namespace SDPatternMatch;
15806
15807 SDLoc DL(Op);
15808
15809 SDValue Op1 = Op.getOperand(i: 0);
15810 SDValue Op2 = Op.getOperand(i: 1);
15811 SDValue Offset = Op.getOperand(i: 2);
15812 SDValue Mask = Op.getOperand(i: 3);
15813 SDValue EVL1 = Op.getOperand(i: 4);
15814 SDValue EVL2 = Op.getOperand(i: 5);
15815
15816 const MVT XLenVT = Subtarget.getXLenVT();
15817 MVT VT = Op.getSimpleValueType();
15818 MVT ContainerVT = VT;
15819 if (VT.isFixedLengthVector()) {
15820 ContainerVT = getContainerForFixedLengthVector(VT);
15821 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
15822 Op2 = convertToScalableVector(VT: ContainerVT, V: Op2, DAG, Subtarget);
15823 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15824 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15825 }
15826
15827 bool IsMaskVector = VT.getVectorElementType() == MVT::i1;
15828 if (IsMaskVector) {
15829 ContainerVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
15830
15831 // Expand input operands
15832 SDValue SplatOneOp1 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
15833 N1: DAG.getUNDEF(VT: ContainerVT),
15834 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL1);
15835 SDValue SplatZeroOp1 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
15836 N1: DAG.getUNDEF(VT: ContainerVT),
15837 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL1);
15838 Op1 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Op1, N2: SplatOneOp1,
15839 N3: SplatZeroOp1, N4: DAG.getUNDEF(VT: ContainerVT), N5: EVL1);
15840
15841 SDValue SplatOneOp2 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
15842 N1: DAG.getUNDEF(VT: ContainerVT),
15843 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL2);
15844 SDValue SplatZeroOp2 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
15845 N1: DAG.getUNDEF(VT: ContainerVT),
15846 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL2);
15847 Op2 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Op2, N2: SplatOneOp2,
15848 N3: SplatZeroOp2, N4: DAG.getUNDEF(VT: ContainerVT), N5: EVL2);
15849 }
15850
15851 auto getVectorFirstEle = [](SDValue Vec) {
15852 SDValue FirstEle;
15853 if (sd_match(N: Vec, P: m_InsertElt(Vec: m_Value(), Val: m_Value(N&: FirstEle), Idx: m_Zero())))
15854 return FirstEle;
15855
15856 if (Vec.getOpcode() == ISD::SPLAT_VECTOR ||
15857 Vec.getOpcode() == ISD::BUILD_VECTOR)
15858 return Vec.getOperand(i: 0);
15859
15860 return SDValue();
15861 };
15862
15863 if (!IsMaskVector && isNullConstant(V: Offset) && isOneConstant(V: EVL1))
15864 if (auto FirstEle = getVectorFirstEle(Op->getOperand(Num: 0))) {
15865 MVT EltVT = ContainerVT.getVectorElementType();
15866 SDValue Result;
15867 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
15868 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
15869 EltVT = EltVT.changeTypeToInteger();
15870 ContainerVT = ContainerVT.changeVectorElementType(EltVT);
15871 Op2 = DAG.getBitcast(VT: ContainerVT, V: Op2);
15872 FirstEle =
15873 DAG.getAnyExtOrTrunc(Op: DAG.getBitcast(VT: EltVT, V: FirstEle), DL, VT: XLenVT);
15874 }
15875 Result = DAG.getNode(Opcode: EltVT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL
15876 : RISCVISD::VSLIDE1UP_VL,
15877 DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Op2,
15878 N3: FirstEle, N4: Mask, N5: EVL2);
15879 Result = DAG.getBitcast(
15880 VT: ContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType()),
15881 V: Result);
15882 return VT.isFixedLengthVector()
15883 ? convertFromScalableVector(VT, V: Result, DAG, Subtarget)
15884 : Result;
15885 }
15886
15887 int64_t ImmValue = cast<ConstantSDNode>(Val&: Offset)->getSExtValue();
15888 SDValue DownOffset, UpOffset;
15889 if (ImmValue >= 0) {
15890 // The operand is a TargetConstant, we need to rebuild it as a regular
15891 // constant.
15892 DownOffset = DAG.getConstant(Val: ImmValue, DL, VT: XLenVT);
15893 UpOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL1, N2: DownOffset);
15894 } else {
15895 // The operand is a TargetConstant, we need to rebuild it as a regular
15896 // constant rather than negating the original operand.
15897 UpOffset = DAG.getConstant(Val: -ImmValue, DL, VT: XLenVT);
15898 DownOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL1, N2: UpOffset);
15899 }
15900
15901 if (ImmValue != 0)
15902 Op1 = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
15903 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Op1, Offset: DownOffset, Mask,
15904 VL: Subtarget.hasVLDependentLatency() ? UpOffset : EVL2);
15905 SDValue Result = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Op1, Op: Op2,
15906 Offset: UpOffset, Mask, VL: EVL2, Policy: RISCVVType::TAIL_AGNOSTIC);
15907
15908 if (IsMaskVector) {
15909 // Truncate Result back to a mask vector (Result has same EVL as Op2)
15910 Result = DAG.getNode(
15911 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT.changeVectorElementType(EltVT: MVT::i1),
15912 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: ContainerVT),
15913 DAG.getCondCode(Cond: ISD::SETNE), DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)),
15914 Mask, EVL2});
15915 }
15916
15917 if (!VT.isFixedLengthVector())
15918 return Result;
15919 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15920}
15921
15922SDValue
15923RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op,
15924 SelectionDAG &DAG) const {
15925 SDLoc DL(Op);
15926 MVT VT = Op.getSimpleValueType();
15927 MVT XLenVT = Subtarget.getXLenVT();
15928
15929 SDValue Op1 = Op.getOperand(i: 0);
15930 SDValue Mask = Op.getOperand(i: 1);
15931 SDValue EVL = Op.getOperand(i: 2);
15932
15933 MVT ContainerVT = VT;
15934 if (VT.isFixedLengthVector()) {
15935 ContainerVT = getContainerForFixedLengthVector(VT);
15936 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
15937 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15938 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15939 }
15940
15941 MVT GatherVT = ContainerVT;
15942 MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger();
15943 // Check if we are working with mask vectors
15944 bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1;
15945 if (IsMaskVector) {
15946 GatherVT = IndicesVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
15947
15948 // Expand input operand
15949 SDValue SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
15950 N1: DAG.getUNDEF(VT: IndicesVT),
15951 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL);
15952 SDValue SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
15953 N1: DAG.getUNDEF(VT: IndicesVT),
15954 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL);
15955 Op1 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: IndicesVT, N1: Op1, N2: SplatOne,
15956 N3: SplatZero, N4: DAG.getUNDEF(VT: IndicesVT), N5: EVL);
15957 }
15958
15959 unsigned EltSize = GatherVT.getScalarSizeInBits();
15960 unsigned MinSize = GatherVT.getSizeInBits().getKnownMinValue();
15961 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
15962 unsigned MaxVLMAX =
15963 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
15964
15965 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
15966 // If this is SEW=8 and VLMAX is unknown or more than 256, we need
15967 // to use vrgatherei16.vv.
15968 // TODO: It's also possible to use vrgatherei16.vv for other types to
15969 // decrease register width for the index calculation.
15970 // NOTE: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
15971 if (MaxVLMAX > 256 && EltSize == 8) {
15972 // If this is LMUL=8, we have to split before using vrgatherei16.vv.
15973 // Split the vector in half and reverse each half using a full register
15974 // reverse.
15975 // Swap the halves and concatenate them.
15976 // Slide the concatenated result by (VLMax - VL).
15977 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
15978 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: GatherVT);
15979 auto [Lo, Hi] = DAG.SplitVector(N: Op1, DL);
15980
15981 SDValue LoRev = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: LoVT, Operand: Lo);
15982 SDValue HiRev = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: HiVT, Operand: Hi);
15983
15984 // Reassemble the low and high pieces reversed.
15985 // NOTE: this Result is unmasked (because we do not need masks for
15986 // shuffles). If in the future this has to change, we can use a SELECT_VL
15987 // between Result and UNDEF using the mask originally passed to VP_REVERSE
15988 SDValue Result =
15989 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: GatherVT, N1: HiRev, N2: LoRev);
15990
15991 // Slide off any elements from past EVL that were reversed into the low
15992 // elements.
15993 SDValue VLMax =
15994 DAG.getElementCount(DL, VT: XLenVT, EC: GatherVT.getVectorElementCount());
15995 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: EVL);
15996
15997 Result = getVSlidedown(DAG, Subtarget, DL, VT: GatherVT,
15998 Passthru: DAG.getUNDEF(VT: GatherVT), Op: Result, Offset: Diff, Mask, VL: EVL);
15999
16000 if (IsMaskVector) {
16001 // Truncate Result back to a mask vector
16002 Result =
16003 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
16004 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: GatherVT),
16005 DAG.getCondCode(Cond: ISD::SETNE),
16006 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), Mask, EVL});
16007 }
16008
16009 if (!VT.isFixedLengthVector())
16010 return Result;
16011 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
16012 }
16013
16014 // Just promote the int type to i16 which will double the LMUL.
16015 IndicesVT = MVT::getVectorVT(VT: MVT::i16, EC: IndicesVT.getVectorElementCount());
16016 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
16017 }
16018
16019 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: IndicesVT, N1: Mask, N2: EVL);
16020 SDValue VecLen =
16021 DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
16022 SDValue VecLenSplat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
16023 N1: DAG.getUNDEF(VT: IndicesVT), N2: VecLen, N3: EVL);
16024 SDValue VRSUB = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: IndicesVT, N1: VecLenSplat, N2: VID,
16025 N3: DAG.getUNDEF(VT: IndicesVT), N4: Mask, N5: EVL);
16026 SDValue Result = DAG.getNode(Opcode: GatherOpc, DL, VT: GatherVT, N1: Op1, N2: VRSUB,
16027 N3: DAG.getUNDEF(VT: GatherVT), N4: Mask, N5: EVL);
16028
16029 if (IsMaskVector) {
16030 // Truncate Result back to a mask vector
16031 Result = DAG.getNode(
16032 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
16033 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: GatherVT), DAG.getCondCode(Cond: ISD::SETNE),
16034 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), Mask, EVL});
16035 }
16036
16037 if (!VT.isFixedLengthVector())
16038 return Result;
16039 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
16040}
16041
16042SDValue RISCVTargetLowering::lowerVPStridedLoad(SDValue Op,
16043 SelectionDAG &DAG) const {
16044 SDLoc DL(Op);
16045 MVT XLenVT = Subtarget.getXLenVT();
16046 MVT VT = Op.getSimpleValueType();
16047 MVT ContainerVT = VT;
16048 if (VT.isFixedLengthVector())
16049 ContainerVT = getContainerForFixedLengthVector(VT);
16050
16051 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
16052
16053 auto *VPNode = cast<VPStridedLoadSDNode>(Val&: Op);
16054 // Check if the mask is known to be all ones
16055 SDValue Mask = VPNode->getMask();
16056 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
16057
16058 SDValue IntID = DAG.getTargetConstant(Val: IsUnmasked ? Intrinsic::riscv_vlse
16059 : Intrinsic::riscv_vlse_mask,
16060 DL, VT: XLenVT);
16061 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID,
16062 DAG.getUNDEF(VT: ContainerVT), VPNode->getBasePtr(),
16063 VPNode->getStride()};
16064 if (!IsUnmasked) {
16065 if (VT.isFixedLengthVector()) {
16066 MVT MaskVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
16067 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
16068 }
16069 Ops.push_back(Elt: Mask);
16070 }
16071 Ops.push_back(Elt: VPNode->getVectorLength());
16072 if (!IsUnmasked) {
16073 SDValue Policy =
16074 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
16075 Ops.push_back(Elt: Policy);
16076 }
16077
16078 SDValue Result =
16079 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
16080 MemVT: VPNode->getMemoryVT(), MMO: VPNode->getMemOperand());
16081 SDValue Chain = Result.getValue(R: 1);
16082
16083 if (VT.isFixedLengthVector())
16084 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
16085
16086 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
16087}
16088
16089SDValue RISCVTargetLowering::lowerVPStridedStore(SDValue Op,
16090 SelectionDAG &DAG) const {
16091 SDLoc DL(Op);
16092 MVT XLenVT = Subtarget.getXLenVT();
16093
16094 auto *VPNode = cast<VPStridedStoreSDNode>(Val&: Op);
16095 SDValue StoreVal = VPNode->getValue();
16096 MVT VT = StoreVal.getSimpleValueType();
16097 MVT ContainerVT = VT;
16098 if (VT.isFixedLengthVector()) {
16099 ContainerVT = getContainerForFixedLengthVector(VT);
16100 StoreVal = convertToScalableVector(VT: ContainerVT, V: StoreVal, DAG, Subtarget);
16101 }
16102
16103 // Check if the mask is known to be all ones
16104 SDValue Mask = VPNode->getMask();
16105 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
16106
16107 SDValue IntID = DAG.getTargetConstant(Val: IsUnmasked ? Intrinsic::riscv_vsse
16108 : Intrinsic::riscv_vsse_mask,
16109 DL, VT: XLenVT);
16110 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID, StoreVal,
16111 VPNode->getBasePtr(), VPNode->getStride()};
16112 if (!IsUnmasked) {
16113 if (VT.isFixedLengthVector()) {
16114 MVT MaskVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
16115 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
16116 }
16117 Ops.push_back(Elt: Mask);
16118 }
16119 Ops.push_back(Elt: VPNode->getVectorLength());
16120
16121 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: VPNode->getVTList(),
16122 Ops, MemVT: VPNode->getMemoryVT(),
16123 MMO: VPNode->getMemOperand());
16124}
16125
16126// Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
16127// matched to a RVV indexed load. The RVV indexed load instructions only
16128// support the "unsigned unscaled" addressing mode; indices are implicitly
16129// zero-extended or truncated to XLEN and are treated as byte offsets. Any
16130// signed or scaled indexing is extended to the XLEN value type and scaled
16131// accordingly.
16132SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
16133 SelectionDAG &DAG) const {
16134 SDLoc DL(Op);
16135 MVT VT = Op.getSimpleValueType();
16136
16137 const auto *MemSD = cast<MemSDNode>(Val: Op.getNode());
16138 EVT MemVT = MemSD->getMemoryVT();
16139 MachineMemOperand *MMO = MemSD->getMemOperand();
16140 SDValue Chain = MemSD->getChain();
16141 SDValue BasePtr = MemSD->getBasePtr();
16142
16143 [[maybe_unused]] ISD::LoadExtType LoadExtType;
16144 SDValue Index, Mask, PassThru, VL;
16145
16146 if (auto *VPGN = dyn_cast<VPGatherSDNode>(Val: Op.getNode())) {
16147 Index = VPGN->getIndex();
16148 Mask = VPGN->getMask();
16149 PassThru = DAG.getUNDEF(VT);
16150 VL = VPGN->getVectorLength();
16151 // VP doesn't support extending loads.
16152 LoadExtType = ISD::NON_EXTLOAD;
16153 } else {
16154 // Else it must be a MGATHER.
16155 auto *MGN = cast<MaskedGatherSDNode>(Val: Op.getNode());
16156 Index = MGN->getIndex();
16157 Mask = MGN->getMask();
16158 PassThru = MGN->getPassThru();
16159 LoadExtType = MGN->getExtensionType();
16160 }
16161
16162 MVT IndexVT = Index.getSimpleValueType();
16163 MVT XLenVT = Subtarget.getXLenVT();
16164
16165 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
16166 "Unexpected VTs!");
16167 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
16168 // Targets have to explicitly opt-in for extending vector loads.
16169 assert(LoadExtType == ISD::NON_EXTLOAD &&
16170 "Unexpected extending MGATHER/VP_GATHER");
16171
16172 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
16173 // the selection of the masked intrinsics doesn't do this for us.
16174 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
16175
16176 MVT ContainerVT = VT;
16177 if (VT.isFixedLengthVector()) {
16178 ContainerVT = getContainerForFixedLengthVector(VT);
16179 IndexVT = MVT::getVectorVT(VT: IndexVT.getVectorElementType(),
16180 EC: ContainerVT.getVectorElementCount());
16181
16182 Index = convertToScalableVector(VT: IndexVT, V: Index, DAG, Subtarget);
16183
16184 if (!IsUnmasked) {
16185 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
16186 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
16187 PassThru = convertToScalableVector(VT: ContainerVT, V: PassThru, DAG, Subtarget);
16188 }
16189 }
16190
16191 if (!VL)
16192 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
16193
16194 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(VT: XLenVT)) {
16195 IndexVT = IndexVT.changeVectorElementType(EltVT: XLenVT);
16196 Index = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IndexVT, Operand: Index);
16197 }
16198
16199 unsigned IntID =
16200 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
16201 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
16202 if (IsUnmasked)
16203 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
16204 else
16205 Ops.push_back(Elt: PassThru);
16206 Ops.push_back(Elt: BasePtr);
16207 Ops.push_back(Elt: Index);
16208 if (!IsUnmasked)
16209 Ops.push_back(Elt: Mask);
16210 Ops.push_back(Elt: VL);
16211 if (!IsUnmasked)
16212 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT));
16213
16214 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
16215 SDValue Result =
16216 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
16217 Chain = Result.getValue(R: 1);
16218
16219 if (VT.isFixedLengthVector())
16220 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
16221
16222 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
16223}
16224
16225// Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
16226// matched to a RVV indexed store. The RVV indexed store instructions only
16227// support the "unsigned unscaled" addressing mode; indices are implicitly
16228// zero-extended or truncated to XLEN and are treated as byte offsets. Any
16229// signed or scaled indexing is extended to the XLEN value type and scaled
16230// accordingly.
16231SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
16232 SelectionDAG &DAG) const {
16233 SDLoc DL(Op);
16234 const auto *MemSD = cast<MemSDNode>(Val: Op.getNode());
16235 EVT MemVT = MemSD->getMemoryVT();
16236 MachineMemOperand *MMO = MemSD->getMemOperand();
16237 SDValue Chain = MemSD->getChain();
16238 SDValue BasePtr = MemSD->getBasePtr();
16239
16240 [[maybe_unused]] bool IsTruncatingStore = false;
16241 SDValue Index, Mask, Val, VL;
16242
16243 if (auto *VPSN = dyn_cast<VPScatterSDNode>(Val: Op.getNode())) {
16244 Index = VPSN->getIndex();
16245 Mask = VPSN->getMask();
16246 Val = VPSN->getValue();
16247 VL = VPSN->getVectorLength();
16248 // VP doesn't support truncating stores.
16249 IsTruncatingStore = false;
16250 } else {
16251 // Else it must be a MSCATTER.
16252 auto *MSN = cast<MaskedScatterSDNode>(Val: Op.getNode());
16253 Index = MSN->getIndex();
16254 Mask = MSN->getMask();
16255 Val = MSN->getValue();
16256 IsTruncatingStore = MSN->isTruncatingStore();
16257 }
16258
16259 MVT VT = Val.getSimpleValueType();
16260 MVT IndexVT = Index.getSimpleValueType();
16261 MVT XLenVT = Subtarget.getXLenVT();
16262
16263 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
16264 "Unexpected VTs!");
16265 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
16266 // Targets have to explicitly opt-in for extending vector loads and
16267 // truncating vector stores.
16268 assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
16269
16270 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
16271 // the selection of the masked intrinsics doesn't do this for us.
16272 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
16273
16274 MVT ContainerVT = VT;
16275 if (VT.isFixedLengthVector()) {
16276 ContainerVT = getContainerForFixedLengthVector(VT);
16277 IndexVT = MVT::getVectorVT(VT: IndexVT.getVectorElementType(),
16278 EC: ContainerVT.getVectorElementCount());
16279
16280 Index = convertToScalableVector(VT: IndexVT, V: Index, DAG, Subtarget);
16281 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
16282
16283 if (!IsUnmasked) {
16284 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
16285 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
16286 }
16287 }
16288
16289 if (!VL)
16290 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
16291
16292 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(VT: XLenVT)) {
16293 IndexVT = IndexVT.changeVectorElementType(EltVT: XLenVT);
16294 Index = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IndexVT, Operand: Index);
16295 }
16296
16297 unsigned IntID =
16298 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
16299 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
16300 Ops.push_back(Elt: Val);
16301 Ops.push_back(Elt: BasePtr);
16302 Ops.push_back(Elt: Index);
16303 if (!IsUnmasked)
16304 Ops.push_back(Elt: Mask);
16305 Ops.push_back(Elt: VL);
16306
16307 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
16308 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
16309}
16310
16311SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
16312 SelectionDAG &DAG) const {
16313 const MVT XLenVT = Subtarget.getXLenVT();
16314 SDLoc DL(Op);
16315 SDValue Chain = Op->getOperand(Num: 0);
16316 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::frm, DL, VT: XLenVT);
16317 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
16318 SDValue RM = DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
16319
16320 // Encoding used for rounding mode in RISC-V differs from that used in
16321 // FLT_ROUNDS. To convert it the RISC-V rounding mode is used as an index in a
16322 // table, which consists of a sequence of 4-bit fields, each representing
16323 // corresponding FLT_ROUNDS mode.
16324 static const int Table =
16325 (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
16326 (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
16327 (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
16328 (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
16329 (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
16330
16331 SDValue Shift =
16332 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: RM, N2: DAG.getConstant(Val: 2, DL, VT: XLenVT));
16333 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT,
16334 N1: DAG.getConstant(Val: Table, DL, VT: XLenVT), N2: Shift);
16335 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Shifted,
16336 N2: DAG.getConstant(Val: 7, DL, VT: XLenVT));
16337
16338 return DAG.getMergeValues(Ops: {Masked, Chain}, dl: DL);
16339}
16340
16341SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
16342 SelectionDAG &DAG) const {
16343 const MVT XLenVT = Subtarget.getXLenVT();
16344 SDLoc DL(Op);
16345 SDValue Chain = Op->getOperand(Num: 0);
16346 SDValue RMValue = Op->getOperand(Num: 1);
16347 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::frm, DL, VT: XLenVT);
16348
16349 // Encoding used for rounding mode in RISC-V differs from that used in
16350 // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
16351 // a table, which consists of a sequence of 4-bit fields, each representing
16352 // corresponding RISC-V mode.
16353 static const unsigned Table =
16354 (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
16355 (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
16356 (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
16357 (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
16358 (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
16359
16360 RMValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: RMValue);
16361
16362 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: RMValue,
16363 N2: DAG.getConstant(Val: 2, DL, VT: XLenVT));
16364 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT,
16365 N1: DAG.getConstant(Val: Table, DL, VT: XLenVT), N2: Shift);
16366 RMValue = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Shifted,
16367 N2: DAG.getConstant(Val: 0x7, DL, VT: XLenVT));
16368 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
16369 N3: RMValue);
16370}
16371
16372SDValue RISCVTargetLowering::lowerGET_FPENV(SDValue Op,
16373 SelectionDAG &DAG) const {
16374 const MVT XLenVT = Subtarget.getXLenVT();
16375 SDLoc DL(Op);
16376 SDValue Chain = Op->getOperand(Num: 0);
16377 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
16378 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
16379 return DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
16380}
16381
16382SDValue RISCVTargetLowering::lowerSET_FPENV(SDValue Op,
16383 SelectionDAG &DAG) const {
16384 const MVT XLenVT = Subtarget.getXLenVT();
16385 SDLoc DL(Op);
16386 SDValue Chain = Op->getOperand(Num: 0);
16387 SDValue EnvValue = Op->getOperand(Num: 1);
16388 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
16389
16390 EnvValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: EnvValue);
16391 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
16392 N3: EnvValue);
16393}
16394
16395SDValue RISCVTargetLowering::lowerRESET_FPENV(SDValue Op,
16396 SelectionDAG &DAG) const {
16397 const MVT XLenVT = Subtarget.getXLenVT();
16398 SDLoc DL(Op);
16399 SDValue Chain = Op->getOperand(Num: 0);
16400 SDValue EnvValue = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
16401 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
16402
16403 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
16404 N3: EnvValue);
16405}
16406
16407const uint64_t ModeMask64 = ~RISCVExceptFlags::ALL;
16408const uint32_t ModeMask32 = ~RISCVExceptFlags::ALL;
16409
16410SDValue RISCVTargetLowering::lowerGET_FPMODE(SDValue Op,
16411 SelectionDAG &DAG) const {
16412 const MVT XLenVT = Subtarget.getXLenVT();
16413 SDLoc DL(Op);
16414 SDValue Chain = Op->getOperand(Num: 0);
16415 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
16416 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
16417 SDValue Result = DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
16418 Chain = Result.getValue(R: 1);
16419 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
16420}
16421
16422SDValue RISCVTargetLowering::lowerSET_FPMODE(SDValue Op,
16423 SelectionDAG &DAG) const {
16424 const MVT XLenVT = Subtarget.getXLenVT();
16425 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
16426 SDLoc DL(Op);
16427 SDValue Chain = Op->getOperand(Num: 0);
16428 SDValue EnvValue = Op->getOperand(Num: 1);
16429 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
16430 SDValue ModeMask = DAG.getConstant(Val: ModeMaskValue, DL, VT: XLenVT);
16431
16432 EnvValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: EnvValue);
16433 EnvValue = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: EnvValue, N2: ModeMask);
16434 Chain = DAG.getNode(Opcode: RISCVISD::CLEAR_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
16435 N3: ModeMask);
16436 return DAG.getNode(Opcode: RISCVISD::SET_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
16437 N3: EnvValue);
16438}
16439
16440SDValue RISCVTargetLowering::lowerRESET_FPMODE(SDValue Op,
16441 SelectionDAG &DAG) const {
16442 const MVT XLenVT = Subtarget.getXLenVT();
16443 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
16444 SDLoc DL(Op);
16445 SDValue Chain = Op->getOperand(Num: 0);
16446 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
16447 SDValue ModeMask = DAG.getConstant(Val: ModeMaskValue, DL, VT: XLenVT);
16448
16449 return DAG.getNode(Opcode: RISCVISD::CLEAR_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
16450 N3: ModeMask);
16451}
16452
16453SDValue RISCVTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
16454 SelectionDAG &DAG) const {
16455 MachineFunction &MF = DAG.getMachineFunction();
16456
16457 bool isRISCV64 = Subtarget.is64Bit();
16458 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
16459
16460 int FI = MF.getFrameInfo().CreateFixedObject(Size: isRISCV64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
16461 return DAG.getFrameIndex(FI, VT: PtrVT);
16462}
16463
16464// Returns the opcode of the target-specific SDNode that implements the 32-bit
16465// form of the given Opcode.
16466static unsigned getRISCVWOpcode(unsigned Opcode) {
16467 switch (Opcode) {
16468 default:
16469 llvm_unreachable("Unexpected opcode");
16470 case ISD::SHL:
16471 return RISCVISD::SLLW;
16472 case ISD::SRA:
16473 return RISCVISD::SRAW;
16474 case ISD::SRL:
16475 return RISCVISD::SRLW;
16476 case ISD::SDIV:
16477 return RISCVISD::DIVW;
16478 case ISD::UDIV:
16479 return RISCVISD::DIVUW;
16480 case ISD::UREM:
16481 return RISCVISD::REMUW;
16482 case ISD::ROTL:
16483 return RISCVISD::ROLW;
16484 case ISD::ROTR:
16485 return RISCVISD::RORW;
16486 }
16487}
16488
16489// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
16490// node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
16491// otherwise be promoted to i64, making it difficult to select the
16492// SLLW/DIVUW/.../*W later one because the fact the operation was originally of
16493// type i8/i16/i32 is lost.
16494static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
16495 unsigned ExtOpc = ISD::ANY_EXTEND) {
16496 SDLoc DL(N);
16497 unsigned WOpcode = getRISCVWOpcode(Opcode: N->getOpcode());
16498 SDValue NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
16499 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16500 SDValue NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
16501 // ReplaceNodeResults requires we maintain the same type for the return value.
16502 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
16503}
16504
16505// Converts the given 32-bit operation to a i64 operation with signed extension
16506// semantic to reduce the signed extension instructions.
16507static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
16508 SDLoc DL(N);
16509 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
16510 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16511 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
16512 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
16513 N2: DAG.getValueType(MVT::i32));
16514 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
16515}
16516
16517// Zero-extend a 32-bit packed vector to the 64-bit packed type WideVT,
16518// clearing the upper lanes.
16519static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
16520 SDValue V, MVT WideVT) {
16521 SDValue Wide =
16522 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: DAG.getBitcast(VT: MVT::i32, V));
16523 return DAG.getBitcast(VT: WideVT, V: Wide);
16524}
16525
16526void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
16527 SmallVectorImpl<SDValue> &Results,
16528 SelectionDAG &DAG) const {
16529 SDLoc DL(N);
16530 switch (N->getOpcode()) {
16531 default:
16532 llvm_unreachable("Don't know how to custom type legalize this operation!");
16533 case ISD::STRICT_FP_TO_SINT:
16534 case ISD::STRICT_FP_TO_UINT:
16535 case ISD::FP_TO_SINT:
16536 case ISD::FP_TO_UINT: {
16537 bool IsStrict = N->isStrictFPOpcode();
16538 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
16539 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
16540 SDValue Op0 = IsStrict ? N->getOperand(Num: 1) : N->getOperand(Num: 0);
16541
16542 // On RV32 only f16 is handled here: it is the only FP type whose finite
16543 // values always fit in an i32 after truncation towards zero, so convert in
16544 // i32 and extend instead of calling __fix(un)hfdi, which compiler-rt does
16545 // not provide. Out of range values and NaN are poison, so the fcvt clamping
16546 // is acceptable. Without Zfh/Zhinx the i32 conversion promotes f16 to f32.
16547 if (!Subtarget.is64Bit()) {
16548 assert(N->getValueType(0) == MVT::i64 &&
16549 "Unexpected custom legalisation");
16550 if (Op0.getValueType() != MVT::f16)
16551 return;
16552 SDValue Cvt;
16553 if (IsStrict) {
16554 Cvt = DAG.getNode(Opcode: IsSigned ? ISD::STRICT_FP_TO_SINT
16555 : ISD::STRICT_FP_TO_UINT,
16556 DL, ResultTys: {MVT::i32, MVT::Other}, Ops: {N->getOperand(Num: 0), Op0});
16557 } else {
16558 Cvt = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL,
16559 VT: MVT::i32, Operand: Op0);
16560 }
16561 Results.push_back(Elt: DAG.getNode(
16562 Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Cvt));
16563 if (IsStrict)
16564 Results.push_back(Elt: Cvt.getValue(R: 1));
16565 return;
16566 }
16567
16568 assert(N->getValueType(0) == MVT::i32 && "Unexpected custom legalisation");
16569 if (getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) !=
16570 TargetLowering::TypeSoftenFloat) {
16571 if (!isTypeLegal(VT: Op0.getValueType()))
16572 return;
16573 if (IsStrict) {
16574 SDValue Chain = N->getOperand(Num: 0);
16575 // In absence of Zfh, promote f16 to f32, then convert.
16576 if (Op0.getValueType() == MVT::f16 &&
16577 !Subtarget.hasStdExtZfhOrZhinx()) {
16578 Op0 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
16579 Ops: {Chain, Op0});
16580 Chain = Op0.getValue(R: 1);
16581 }
16582 unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
16583 : RISCVISD::STRICT_FCVT_WU_RV64;
16584 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other);
16585 SDValue Res = DAG.getNode(
16586 Opcode: Opc, DL, VTList: VTs, N1: Chain, N2: Op0,
16587 N3: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: MVT::i64));
16588 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16589 Results.push_back(Elt: Res.getValue(R: 1));
16590 return;
16591 }
16592 // For bf16, or f16 in absence of Zfh, promote [b]f16 to f32 and then
16593 // convert.
16594 if ((Op0.getValueType() == MVT::f16 &&
16595 !Subtarget.hasStdExtZfhOrZhinx()) ||
16596 Op0.getValueType() == MVT::bf16)
16597 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
16598
16599 unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
16600 SDValue Res =
16601 DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: Op0,
16602 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: MVT::i64));
16603 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16604 return;
16605 }
16606 // If the FP type needs to be softened, emit a library call using the 'si'
16607 // version. If we left it to default legalization we'd end up with 'di'. If
16608 // the FP type doesn't need to be softened just let generic type
16609 // legalization promote the result type.
16610 RTLIB::Libcall LC;
16611 if (IsSigned)
16612 LC = RTLIB::getFPTOSINT(OpVT: Op0.getValueType(), RetVT: N->getValueType(ResNo: 0));
16613 else
16614 LC = RTLIB::getFPTOUINT(OpVT: Op0.getValueType(), RetVT: N->getValueType(ResNo: 0));
16615 MakeLibCallOptions CallOptions;
16616 EVT OpVT = Op0.getValueType();
16617 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
16618 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
16619 SDValue Result;
16620 std::tie(args&: Result, args&: Chain) =
16621 makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0), Ops: Op0, CallOptions, dl: DL, Chain);
16622 Results.push_back(Elt: Result);
16623 if (IsStrict)
16624 Results.push_back(Elt: Chain);
16625 break;
16626 }
16627 case ISD::LROUND: {
16628 SDValue Op0 = N->getOperand(Num: 0);
16629 EVT Op0VT = Op0.getValueType();
16630 if (getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) !=
16631 TargetLowering::TypeSoftenFloat) {
16632 if (!isTypeLegal(VT: Op0VT))
16633 return;
16634
16635 // In absence of Zfh, promote f16 to f32, then convert.
16636 if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx())
16637 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
16638
16639 SDValue Res =
16640 DAG.getNode(Opcode: RISCVISD::FCVT_W_RV64, DL, VT: MVT::i64, N1: Op0,
16641 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RMM, DL, VT: MVT::i64));
16642 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16643 return;
16644 }
16645 // If the FP type needs to be softened, emit a library call to lround. We'll
16646 // need to truncate the result. We assume any value that doesn't fit in i32
16647 // is allowed to return an unspecified value.
16648 RTLIB::Libcall LC = RTLIB::getLROUND(VT: Op0.getValueType());
16649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected FP type for LROUND!");
16650 MakeLibCallOptions CallOptions;
16651 EVT OpVT = Op0.getValueType();
16652 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
16653 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
16654 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
16655 Results.push_back(Elt: Result);
16656 break;
16657 }
16658 case ISD::READCYCLECOUNTER:
16659 case ISD::READSTEADYCOUNTER: {
16660 assert(!Subtarget.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
16661 "has custom type legalization on riscv32");
16662
16663 SDValue LoCounter, HiCounter;
16664 MVT XLenVT = Subtarget.getXLenVT();
16665 if (N->getOpcode() == ISD::READCYCLECOUNTER) {
16666 LoCounter = DAG.getTargetConstant(Val: RISCVSysReg::cycle, DL, VT: XLenVT);
16667 HiCounter = DAG.getTargetConstant(Val: RISCVSysReg::cycleh, DL, VT: XLenVT);
16668 } else {
16669 LoCounter = DAG.getTargetConstant(Val: RISCVSysReg::time, DL, VT: XLenVT);
16670 HiCounter = DAG.getTargetConstant(Val: RISCVSysReg::timeh, DL, VT: XLenVT);
16671 }
16672 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
16673 SDValue RCW = DAG.getNode(Opcode: RISCVISD::READ_COUNTER_WIDE, DL, VTList: VTs,
16674 N1: N->getOperand(Num: 0), N2: LoCounter, N3: HiCounter);
16675
16676 Results.push_back(
16677 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: RCW, N2: RCW.getValue(R: 1)));
16678 Results.push_back(Elt: RCW.getValue(R: 2));
16679 break;
16680 }
16681 case ISD::LOAD: {
16682 if (!ISD::isNON_EXTLoad(N))
16683 return;
16684
16685 // Use a SEXTLOAD instead of the default EXTLOAD. Similar to the
16686 // sext_inreg we emit for ADD/SUB/MUL/SLLI.
16687 LoadSDNode *Ld = cast<LoadSDNode>(Val: N);
16688
16689 if (N->getValueType(ResNo: 0) == MVT::i64) {
16690 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
16691 "Unexpected custom legalisation");
16692
16693 if (Ld->getAlign() < Subtarget.getZilsdAlign())
16694 return;
16695
16696 SDLoc DL(N);
16697 SDValue Result = DAG.getMemIntrinsicNode(
16698 Opcode: RISCVISD::LD_RV32, dl: DL,
16699 VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32, MVT::Other}),
16700 Ops: {Ld->getChain(), Ld->getBasePtr()}, MemVT: MVT::i64, MMO: Ld->getMemOperand());
16701 SDValue Lo = Result.getValue(R: 0);
16702 SDValue Hi = Result.getValue(R: 1);
16703 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Lo, N2: Hi);
16704 Results.append(IL: {Pair, Result.getValue(R: 2)});
16705 return;
16706 }
16707
16708 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
16709 "Unexpected custom legalisation");
16710
16711 SDLoc dl(N);
16712 SDValue Res = DAG.getExtLoad(ExtType: ISD::SEXTLOAD, dl, VT: MVT::i64, Chain: Ld->getChain(),
16713 Ptr: Ld->getBasePtr(), MemVT: Ld->getMemoryVT(),
16714 MMO: Ld->getMemOperand());
16715 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Res));
16716 Results.push_back(Elt: Res.getValue(R: 1));
16717 return;
16718 }
16719 case ISD::MUL: {
16720 unsigned Size = N->getSimpleValueType(ResNo: 0).getSizeInBits();
16721 unsigned XLen = Subtarget.getXLen();
16722 if (Size > XLen) {
16723 // This multiply needs to be expanded, try to use MULH+MUL or WMUL if
16724 // possible. We duplicate the default legalization to
16725 // MULHU/MULHS/UMUL_LOHI/SMUL_LOHI to minimize the number of calls to
16726 // MaskedValueIsZero and ComputeNumSignBits
16727 // FIXME: Should we have a target independent MULHSU/WMULSU node? Are
16728 // there are other targets that could use it?
16729 assert(Size == (XLen * 2) && "Unexpected custom legalisation");
16730
16731 auto MakeMULPair = [&](SDValue L, SDValue R, unsigned HighOpc,
16732 unsigned LoHiOpc) {
16733 MVT XLenVT = Subtarget.getXLenVT();
16734 L = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: XLenVT, Operand: L);
16735 R = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: XLenVT, Operand: R);
16736 SDValue Lo, Hi;
16737 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
16738 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32);
16739 Lo = DAG.getNode(Opcode: LoHiOpc, DL, VTList: VTs, N1: L, N2: R);
16740 Hi = Lo.getValue(R: 1);
16741 } else {
16742 Lo = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: L, N2: R);
16743 Hi = DAG.getNode(Opcode: HighOpc, DL, VT: XLenVT, N1: L, N2: R);
16744 }
16745 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
16746 };
16747
16748 SDValue LHS = N->getOperand(Num: 0);
16749 SDValue RHS = N->getOperand(Num: 1);
16750
16751 APInt HighMask = APInt::getHighBitsSet(numBits: Size, hiBitsSet: XLen);
16752 bool LHSIsU = DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask);
16753 bool RHSIsU = DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask);
16754 if (LHSIsU && RHSIsU) {
16755 Results.push_back(Elt: MakeMULPair(LHS, RHS, ISD::MULHU, ISD::UMUL_LOHI));
16756 return;
16757 }
16758
16759 bool LHSIsS = DAG.ComputeNumSignBits(Op: LHS) > XLen;
16760 bool RHSIsS = DAG.ComputeNumSignBits(Op: RHS) > XLen;
16761 if (LHSIsS && RHSIsS)
16762 Results.push_back(Elt: MakeMULPair(LHS, RHS, ISD::MULHS, ISD::SMUL_LOHI));
16763 else if (RHSIsU && LHSIsS)
16764 Results.push_back(
16765 Elt: MakeMULPair(LHS, RHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
16766 else if (LHSIsU && RHSIsS)
16767 Results.push_back(
16768 Elt: MakeMULPair(RHS, LHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
16769
16770 return;
16771 }
16772 [[fallthrough]];
16773 }
16774 case ISD::ADD:
16775 case ISD::SUB:
16776 if (N->getValueType(ResNo: 0) == MVT::i64) {
16777 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
16778 "Unexpected custom legalisation");
16779
16780 // Expand to ADDD/SUBD.
16781 auto [LHSLo, LHSHi] =
16782 DAG.SplitScalar(N: N->getOperand(Num: 0), DL, LoVT: MVT::i32, HiVT: MVT::i32);
16783 auto [RHSLo, RHSHi] =
16784 DAG.SplitScalar(N: N->getOperand(Num: 1), DL, LoVT: MVT::i32, HiVT: MVT::i32);
16785 unsigned Opc =
16786 N->getOpcode() == ISD::ADD ? RISCVISD::ADDD : RISCVISD::SUBD;
16787 SDValue Res = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
16788 N1: LHSLo, N2: LHSHi, N3: RHSLo, N4: RHSHi);
16789 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Res, N2: Res.getValue(R: 1));
16790 Results.push_back(Elt: Res);
16791 return;
16792 }
16793
16794 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
16795 "Unexpected custom legalisation");
16796 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
16797 break;
16798 case ISD::TRUNCATE: {
16799 MVT VT = N->getSimpleValueType(ResNo: 0);
16800 assert(VT.isFixedLengthVector() && Subtarget.hasStdExtP() &&
16801 Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
16802 "Unexpected custom legalisation");
16803
16804 // v4i16->v4i8 and v2i32->v2i16 truncates aren't legal on their own, but
16805 // the widened result type is. Bitcast the operand to the widened result
16806 // type and use a shuffle to select the low half of each element, which
16807 // gets matched to a P-extension packed narrowing convert
16808 // (unzip8p/unzip16p).
16809
16810 SDValue Op0 = N->getOperand(Num: 0);
16811
16812 // Input should be a 64-bit vector.
16813 if (!Op0.getValueType().is64BitVector())
16814 break;
16815
16816 MVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT).getSimpleVT();
16817
16818 unsigned NumElts = VT.getVectorNumElements();
16819 SmallVector<int, 8> ShuffleMask(WidenVT.getVectorNumElements(), -1);
16820 for (unsigned i = 0; i != NumElts; ++i)
16821 ShuffleMask[i] = i * 2;
16822
16823 SDValue Src = DAG.getBitcast(VT: WidenVT, V: Op0);
16824 Results.push_back(Elt: DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: Src,
16825 N2: DAG.getUNDEF(VT: WidenVT), Mask: ShuffleMask));
16826 return;
16827 }
16828 case ISD::SHL:
16829 case ISD::SRA:
16830 case ISD::SRL: {
16831 EVT VT = N->getValueType(ResNo: 0);
16832 if (VT.isFixedLengthVector() && Subtarget.hasStdExtP()) {
16833 assert(Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
16834 "Unexpected vector type for P-extension shift");
16835
16836 // If shift amount is a splat, don't scalarize - let normal widening
16837 // and SIMD patterns handle it (pslli.h, psrli.h, etc.)
16838 SDValue ShiftAmt = N->getOperand(Num: 1);
16839 if (DAG.isSplatValue(V: ShiftAmt, /*AllowUndefs=*/true))
16840 break;
16841
16842 EVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
16843 unsigned WidenNumElts = WidenVT.getVectorNumElements();
16844 // Unroll with OrigNumElts operations, padding result to WidenNumElts
16845 SDValue Res = DAG.UnrollVectorOp(N, ResNE: WidenNumElts);
16846 Results.push_back(Elt: Res);
16847 break;
16848 }
16849
16850 if (VT == MVT::i64) {
16851 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
16852 "Unexpected custom legalisation");
16853
16854 SDValue LHS = N->getOperand(Num: 0);
16855 SDValue ShAmt = N->getOperand(Num: 1);
16856
16857 unsigned WideOpc = 0;
16858 APInt HighMask = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
16859 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask))
16860 WideOpc = RISCVISD::WSLL;
16861 else if (DAG.ComputeMaxSignificantBits(Op: LHS) <= 32)
16862 WideOpc = RISCVISD::WSLA;
16863
16864 if (WideOpc) {
16865 SDValue Res =
16866 DAG.getNode(Opcode: WideOpc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
16867 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LHS),
16868 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: ShAmt));
16869 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: N->getValueType(ResNo: 0),
16870 N1: Res, N2: Res.getValue(R: 1)));
16871 return;
16872 }
16873
16874 // Only handle constant shifts < 32. Non-constant shifts are handled by
16875 // lowerShiftLeftParts/lowerShiftRightParts, and shifts >= 32 use default
16876 // legalization.
16877 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
16878 if (!ShAmtC || ShAmtC->getZExtValue() >= 32)
16879 break;
16880
16881 auto [Lo, Hi] = DAG.SplitScalar(N: LHS, DL, LoVT: MVT::i32, HiVT: MVT::i32);
16882
16883 // If the shift amount operand is coming from a vector legalization it may
16884 // have an illegal type.
16885 if (ShAmt.getValueType() != MVT::i32)
16886 ShAmt = DAG.getZExtOrTrunc(Op: ShAmt, DL, VT: MVT::i32);
16887
16888 SDValue LoRes, HiRes;
16889 if (N->getOpcode() == ISD::SHL) {
16890 // Lo = slli Lo, shamt
16891 // Hi = nsrli {Hi, Lo}, (32 - shamt)
16892 uint64_t ShAmtVal = ShAmtC->getZExtValue();
16893 LoRes = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Lo, N2: ShAmt);
16894 HiRes = DAG.getNode(Opcode: RISCVISD::NSRL, DL, VT: MVT::i32, N1: Lo, N2: Hi,
16895 N3: DAG.getConstant(Val: 32 - ShAmtVal, DL, VT: MVT::i32));
16896 } else {
16897 bool IsSRA = N->getOpcode() == ISD::SRA;
16898 LoRes = DAG.getNode(Opcode: IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL,
16899 VT: MVT::i32, N1: Lo, N2: Hi, N3: ShAmt);
16900 HiRes =
16901 DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32, N1: Hi, N2: ShAmt);
16902 }
16903 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: LoRes, N2: HiRes);
16904 Results.push_back(Elt: Res);
16905 return;
16906 }
16907
16908 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
16909 "Unexpected custom legalisation");
16910 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
16911 // If we can use a BSET instruction, allow default promotion to apply.
16912 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
16913 isOneConstant(V: N->getOperand(Num: 0)))
16914 break;
16915 Results.push_back(Elt: customLegalizeToWOp(N, DAG));
16916 break;
16917 }
16918
16919 // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
16920 // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
16921 // shift amount.
16922 if (N->getOpcode() == ISD::SHL) {
16923 SDLoc DL(N);
16924 SDValue NewOp0 =
16925 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
16926 SDValue NewOp1 =
16927 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16928 SDValue NewWOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
16929 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
16930 N2: DAG.getValueType(MVT::i32));
16931 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes));
16932 }
16933
16934 break;
16935 }
16936 case ISD::ROTL:
16937 case ISD::ROTR:
16938 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
16939 "Unexpected custom legalisation");
16940 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
16941 Subtarget.hasVendorXTHeadBb()) &&
16942 "Unexpected custom legalization");
16943 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
16944 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
16945 return;
16946 Results.push_back(Elt: customLegalizeToWOp(N, DAG));
16947 break;
16948 case ISD::CTTZ:
16949 case ISD::CTTZ_ZERO_POISON:
16950 case ISD::CTLZ:
16951 case ISD::CTLZ_ZERO_POISON:
16952 case ISD::CTLS: {
16953 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
16954 "Unexpected custom legalisation");
16955
16956 SDValue NewOp0 =
16957 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
16958 unsigned Opc;
16959 switch (N->getOpcode()) {
16960 default: llvm_unreachable("Unexpected opcode");
16961 case ISD::CTTZ:
16962 case ISD::CTTZ_ZERO_POISON:
16963 Opc = RISCVISD::CTZW;
16964 break;
16965 case ISD::CTLZ:
16966 case ISD::CTLZ_ZERO_POISON:
16967 Opc = RISCVISD::CLZW;
16968 break;
16969 case ISD::CTLS:
16970 Opc = RISCVISD::CLSW;
16971 break;
16972 }
16973
16974 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, Operand: NewOp0);
16975 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16976 return;
16977 }
16978 case ISD::SDIV:
16979 case ISD::UDIV:
16980 case ISD::UREM: {
16981 MVT VT = N->getSimpleValueType(ResNo: 0);
16982 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
16983 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
16984 "Unexpected custom legalisation");
16985 // Don't promote division/remainder by constant since we should expand those
16986 // to multiply by magic constant.
16987 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
16988 if (N->getOperand(Num: 1).getOpcode() == ISD::Constant &&
16989 !isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
16990 return;
16991
16992 // If the input is i32, use ANY_EXTEND since the W instructions don't read
16993 // the upper 32 bits. For other types we need to sign or zero extend
16994 // based on the opcode.
16995 unsigned ExtOpc = ISD::ANY_EXTEND;
16996 if (VT != MVT::i32)
16997 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
16998 : ISD::ZERO_EXTEND;
16999
17000 Results.push_back(Elt: customLegalizeToWOp(N, DAG, ExtOpc));
17001 break;
17002 }
17003 case ISD::SADDO:
17004 case ISD::SSUBO: {
17005 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
17006 "Unexpected custom legalisation");
17007
17008 // This is similar to the default legalization, but we return the
17009 // sext_inreg instead of the add/sub.
17010 bool IsAdd = N->getOpcode() == ISD::SADDO;
17011 SDValue LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
17012 SDValue RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17013 SDValue Op =
17014 DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT: MVT::i64, N1: LHS, N2: RHS);
17015 SDValue Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Op,
17016 N2: DAG.getValueType(MVT::i32));
17017
17018 SDValue Overflow;
17019
17020 // If the RHS is a constant, we can simplify ConditionRHS below. Otherwise
17021 // use the default legalization.
17022 if (IsAdd && isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
17023 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
17024
17025 // For an addition, the result should be less than one of the operands
17026 // (LHS) if and only if the other operand (RHS) is negative, otherwise
17027 // there will be overflow.
17028 EVT OType = N->getValueType(ResNo: 1);
17029 SDValue ResultLowerThanLHS =
17030 DAG.getSetCC(DL, VT: OType, LHS: Res, RHS: LHS, Cond: ISD::SETLT);
17031 SDValue ConditionRHS = DAG.getSetCC(DL, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
17032
17033 Overflow =
17034 DAG.getNode(Opcode: ISD::XOR, DL, VT: OType, N1: ConditionRHS, N2: ResultLowerThanLHS);
17035 } else {
17036 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res, RHS: Op, Cond: ISD::SETNE);
17037 }
17038
17039 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17040 Results.push_back(Elt: Overflow);
17041 return;
17042 }
17043 case ISD::UADDO:
17044 case ISD::USUBO: {
17045 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
17046 "Unexpected custom legalisation");
17047 bool IsAdd = N->getOpcode() == ISD::UADDO;
17048 // Create an ADDW or SUBW.
17049 SDValue LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
17050 SDValue RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17051 SDValue Res =
17052 DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT: MVT::i64, N1: LHS, N2: RHS);
17053 Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Res,
17054 N2: DAG.getValueType(MVT::i32));
17055
17056 SDValue Overflow;
17057 if (IsAdd && isOneConstant(V: RHS)) {
17058 // Special case uaddo X, 1 overflowed if the addition result is 0.
17059 // The general case (X + C) < C is not necessarily beneficial. Although we
17060 // reduce the live range of X, we may introduce the materialization of
17061 // constant C, especially when the setcc result is used by branch. We have
17062 // no compare with constant and branch instructions.
17063 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res,
17064 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i64), Cond: ISD::SETEQ);
17065 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
17066 // Special case uaddo X, -1 overflowed if X != 0.
17067 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: N->getOperand(Num: 0),
17068 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Cond: ISD::SETNE);
17069 } else {
17070 // Sign extend the LHS and perform an unsigned compare with the ADDW
17071 // result. Since the inputs are sign extended from i32, this is equivalent
17072 // to comparing the lower 32 bits.
17073 LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
17074 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res, RHS: LHS,
17075 Cond: IsAdd ? ISD::SETULT : ISD::SETUGT);
17076 }
17077
17078 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17079 Results.push_back(Elt: Overflow);
17080 return;
17081 }
17082 case ISD::UADDSAT:
17083 case ISD::USUBSAT:
17084 case ISD::SADDSAT:
17085 case ISD::SSUBSAT: {
17086 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
17087 "Unexpected custom legalisation");
17088
17089 if (Subtarget.hasStdExtP()) {
17090 // On RV64, map scalar i32 saturating add/sub through lane 0 of a packed
17091 // v2i32 operation so we can select ps*.w instructions.
17092 SDValue LHS = DAG.getNode(
17093 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
17094 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0)));
17095 SDValue RHS = DAG.getNode(
17096 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
17097 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1)));
17098 SDValue VecRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v2i32, N1: LHS, N2: RHS);
17099 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
17100 Results.push_back(
17101 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: VecRes, N2: Zero));
17102 return;
17103 }
17104
17105 assert(!Subtarget.hasStdExtZbb() && "Unexpected custom legalisation");
17106 Results.push_back(Elt: expandAddSubSat(Node: N, DAG));
17107 return;
17108 }
17109 case ISD::ABS:
17110 case ISD::ABS_MIN_POISON: {
17111 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
17112 "Unexpected custom legalisation");
17113
17114 if (Subtarget.hasStdExtP()) {
17115 SDValue Src =
17116 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
17117 SDValue Abs = DAG.getNode(Opcode: RISCVISD::ABSW, DL, VT: MVT::i64, Operand: Src);
17118 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Abs));
17119 return;
17120 }
17121
17122 if (Subtarget.hasStdExtZbb()) {
17123 // Emit a special node that will be expanded to NEGW+MAX at isel.
17124 // This allows us to remember that the result is sign extended. Expanding
17125 // to NEGW+MAX here requires a Freeze which breaks ComputeNumSignBits.
17126 SDValue Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64,
17127 Operand: N->getOperand(Num: 0));
17128 SDValue Abs = DAG.getNode(Opcode: RISCVISD::NEGW_MAX, DL, VT: MVT::i64, Operand: Src);
17129 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Abs));
17130 return;
17131 }
17132
17133 // Expand abs to Y = (sraiw X, 31); subw(xor(X, Y), Y)
17134 SDValue Src = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
17135
17136 // Freeze the source so we can increase it's use count.
17137 Src = DAG.getFreeze(V: Src);
17138
17139 // Copy sign bit to all bits using the sraiw pattern.
17140 SDValue SignFill = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Src,
17141 N2: DAG.getValueType(MVT::i32));
17142 SignFill = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i64, N1: SignFill,
17143 N2: DAG.getConstant(Val: 31, DL, VT: MVT::i64));
17144
17145 SDValue NewRes = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Src, N2: SignFill);
17146 NewRes = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: NewRes, N2: SignFill);
17147
17148 // NOTE: The result is only required to be anyextended, but sext is
17149 // consistent with type legalization of sub.
17150 NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewRes,
17151 N2: DAG.getValueType(MVT::i32));
17152 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes));
17153 return;
17154 }
17155 case ISD::BITCAST: {
17156 EVT VT = N->getValueType(ResNo: 0);
17157 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
17158 SDValue Op0 = N->getOperand(Num: 0);
17159 EVT Op0VT = Op0.getValueType();
17160 MVT XLenVT = Subtarget.getXLenVT();
17161 if (VT == MVT::i16 &&
17162 ((Op0VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
17163 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
17164 SDValue FPConv = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Op0);
17165 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FPConv));
17166 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
17167 Subtarget.hasStdExtFOrZfinx()) {
17168 SDValue FPConv =
17169 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Op0);
17170 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: FPConv));
17171 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() &&
17172 Subtarget.hasStdExtDOrZdinx()) {
17173 SDValue NewReg = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
17174 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Op0);
17175 SDValue Lo = NewReg.getValue(R: 0);
17176 SDValue Hi = NewReg.getValue(R: 1);
17177 // For big-endian, swap the order when building the i64 pair.
17178 if (!Subtarget.isLittleEndian())
17179 std::swap(a&: Lo, b&: Hi);
17180 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Lo, N2: Hi);
17181 Results.push_back(Elt: RetReg);
17182 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
17183 isTypeLegal(VT: Op0VT)) {
17184 // Custom-legalize bitcasts from fixed-length vector types to illegal
17185 // scalar types in order to improve codegen. Bitcast the vector to a
17186 // one-element vector type whose element type is the same as the result
17187 // type, and extract the first element.
17188 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 1);
17189 if (isTypeLegal(VT: BVT)) {
17190 SDValue BVec = DAG.getBitcast(VT: BVT, V: Op0);
17191 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT, Vec: BVec, Idx: 0));
17192 }
17193 }
17194 break;
17195 }
17196 case ISD::BITREVERSE: {
17197 assert(N->getValueType(0) == MVT::i8 && Subtarget.hasStdExtZbkb() &&
17198 "Unexpected custom legalisation");
17199 MVT XLenVT = Subtarget.getXLenVT();
17200 SDValue NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 0));
17201 SDValue NewRes = DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT: XLenVT, Operand: NewOp);
17202 // ReplaceNodeResults requires we maintain the same type for the return
17203 // value.
17204 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: NewRes));
17205 break;
17206 }
17207 case RISCVISD::BREV8:
17208 case RISCVISD::ORC_B: {
17209 MVT VT = N->getSimpleValueType(ResNo: 0);
17210 MVT XLenVT = Subtarget.getXLenVT();
17211 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
17212 "Unexpected custom legalisation");
17213 assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) ||
17214 (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) &&
17215 "Unexpected extension");
17216 SDValue NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 0));
17217 SDValue NewRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: XLenVT, Operand: NewOp);
17218 // ReplaceNodeResults requires we maintain the same type for the return
17219 // value.
17220 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewRes));
17221 break;
17222 }
17223 case RISCVISD::ASUB:
17224 case RISCVISD::ASUBU:
17225 case RISCVISD::MULHSU:
17226 case RISCVISD::MULHR:
17227 case RISCVISD::MULHRU:
17228 case RISCVISD::MULHRSU: {
17229 MVT VT = N->getSimpleValueType(ResNo: 0);
17230 SDValue Op0 = N->getOperand(Num: 0);
17231 SDValue Op1 = N->getOperand(Num: 1);
17232 unsigned Opcode = N->getOpcode();
17233 // PMULH* variants don't support i8
17234 [[maybe_unused]] bool IsMulH =
17235 Opcode == RISCVISD::MULHSU || Opcode == RISCVISD::MULHR ||
17236 Opcode == RISCVISD::MULHRU || Opcode == RISCVISD::MULHRSU;
17237 assert(VT == MVT::v2i16 || (!IsMulH && VT == MVT::v4i8));
17238 MVT NewVT = MVT::v4i16;
17239 if (VT == MVT::v4i8)
17240 NewVT = MVT::v8i8;
17241 SDValue Undef = DAG.getUNDEF(VT);
17242 Op0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, Ops: {Op0, Undef});
17243 Op1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, Ops: {Op1, Undef});
17244 Results.push_back(Elt: DAG.getNode(Opcode, DL, VT: NewVT, Ops: {Op0, Op1}));
17245 return;
17246 }
17247 case ISD::EXTRACT_VECTOR_ELT: {
17248 // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
17249 // type is illegal (currently only vXi64 RV32).
17250 // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
17251 // transferred to the destination register. We issue two of these from the
17252 // upper- and lower- halves of the SEW-bit vector element, slid down to the
17253 // first element.
17254 SDValue Vec = N->getOperand(Num: 0);
17255 SDValue Idx = N->getOperand(Num: 1);
17256
17257 // The vector type hasn't been legalized yet so we can't issue target
17258 // specific nodes if it needs legalization.
17259 // FIXME: We would manually legalize if it's important.
17260 if (!isTypeLegal(VT: Vec.getValueType()))
17261 return;
17262
17263 MVT VecVT = Vec.getSimpleValueType();
17264
17265 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
17266 VecVT.getVectorElementType() == MVT::i64 &&
17267 "Unexpected EXTRACT_VECTOR_ELT legalization");
17268
17269 // If this is a fixed vector, we need to convert it to a scalable vector.
17270 MVT ContainerVT = VecVT;
17271 if (VecVT.isFixedLengthVector()) {
17272 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
17273 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
17274 }
17275
17276 MVT XLenVT = Subtarget.getXLenVT();
17277
17278 // Use a VL of 1 to avoid processing more elements than we need.
17279 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT, DL, DAG, Subtarget);
17280
17281 // Unless the index is known to be 0, we must slide the vector down to get
17282 // the desired element into index 0.
17283 if (!isNullConstant(V: Idx)) {
17284 Vec = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
17285 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: Idx, Mask, VL);
17286 }
17287
17288 // Extract the lower XLEN bits of the correct vector element.
17289 SDValue EltLo = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
17290
17291 // To extract the upper XLEN bits of the vector element, shift the first
17292 // element right by 32 bits and re-extract the lower XLEN bits.
17293 SDValue ThirtyTwoV = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
17294 N1: DAG.getUNDEF(VT: ContainerVT),
17295 N2: DAG.getConstant(Val: 32, DL, VT: XLenVT), N3: VL);
17296 SDValue LShr32 =
17297 DAG.getNode(Opcode: RISCVISD::SRL_VL, DL, VT: ContainerVT, N1: Vec, N2: ThirtyTwoV,
17298 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
17299
17300 SDValue EltHi = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: LShr32);
17301
17302 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: EltLo, N2: EltHi));
17303 break;
17304 }
17305 case ISD::INTRINSIC_WO_CHAIN: {
17306 unsigned IntNo = N->getConstantOperandVal(Num: 0);
17307 switch (IntNo) {
17308 default:
17309 llvm_unreachable(
17310 "Don't know how to custom type legalize this intrinsic!");
17311 case Intrinsic::experimental_get_vector_length: {
17312 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget);
17313 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17314 return;
17315 }
17316 case Intrinsic::riscv_psext_h:
17317 reportFatalUsageError(reason: "unsupported llvm.riscv.psext intrinsic");
17318 case Intrinsic::riscv_pzext_h:
17319 reportFatalUsageError(reason: "unsupported llvm.riscv.pzext intrinsic");
17320 case Intrinsic::riscv_psext_b:
17321 case Intrinsic::riscv_pzext_b: {
17322 bool IsSExt = IntNo == Intrinsic::riscv_psext_b;
17323 const char *UnsupportedMsg =
17324 IsSExt ? "unsupported llvm.riscv.psext intrinsic"
17325 : "unsupported llvm.riscv.pzext intrinsic";
17326 EVT VT = N->getValueType(ResNo: 0);
17327 if (!Subtarget.is64Bit() || VT != MVT::v2i16)
17328 reportFatalUsageError(reason: UnsupportedMsg);
17329
17330 SDValue Src = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16,
17331 N1: N->getOperand(Num: 1), N2: DAG.getUNDEF(VT));
17332 SDValue Res;
17333 if (IsSExt)
17334 Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::v4i16, N1: Src,
17335 N2: DAG.getValueType(MVT::v4i8));
17336 else
17337 Res = lowerPZExt(Src, DL, DAG, Subtarget);
17338 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17339 return;
17340 }
17341 case Intrinsic::riscv_pmhacc:
17342 case Intrinsic::riscv_pmhracc:
17343 case Intrinsic::riscv_pmhaccu:
17344 case Intrinsic::riscv_pmhraccu:
17345 case Intrinsic::riscv_pmhaccsu:
17346 case Intrinsic::riscv_pmhraccsu: {
17347 EVT VT = N->getValueType(ResNo: 0);
17348 if (!Subtarget.is64Bit() || VT != MVT::v2i16)
17349 return;
17350
17351 EVT WideVT = MVT::v4i16;
17352 SDValue Undef = DAG.getUNDEF(VT);
17353 SDValue Rd =
17354 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT, N1: N->getOperand(Num: 1), N2: Undef);
17355 SDValue Rs1 =
17356 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT, N1: N->getOperand(Num: 2), N2: Undef);
17357 SDValue Rs2 =
17358 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT, N1: N->getOperand(Num: 3), N2: Undef);
17359 SDValue Res = DAG.getNode(Opcode: getRVPMulHighAccumulateOpcode(IntNo), DL,
17360 VT: WideVT, N1: Rd, N2: Rs1, N3: Rs2);
17361 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17362 return;
17363 }
17364 case Intrinsic::riscv_pm4add:
17365 case Intrinsic::riscv_pm2add:
17366 case Intrinsic::riscv_pm2add_x:
17367 case Intrinsic::riscv_pm4addu:
17368 case Intrinsic::riscv_pm2addu:
17369 case Intrinsic::riscv_pmq2add:
17370 case Intrinsic::riscv_pmqr2add:
17371 case Intrinsic::riscv_pm2sadd:
17372 case Intrinsic::riscv_pm2sadd_x:
17373 case Intrinsic::riscv_pm2sub:
17374 case Intrinsic::riscv_pm2sub_x:
17375 case Intrinsic::riscv_pm4addsu:
17376 case Intrinsic::riscv_pm2addsu: {
17377 MVT VT = N->getSimpleValueType(ResNo: 0);
17378 unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
17379 SDValue Rs1 = N->getOperand(Num: 1);
17380 SDValue Rs2 = N->getOperand(Num: 2);
17381
17382 if (!Subtarget.is64Bit() && VT == MVT::i64) {
17383 SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG);
17384 Results.push_back(Elt: Res);
17385 return;
17386 }
17387
17388 assert(Subtarget.is64Bit() && VT == MVT::i32 &&
17389 "Unexpected horizontal multiply legalization");
17390 MVT SrcVT = Rs1.getSimpleValueType();
17391 MVT WideSrcVT = SrcVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
17392 Rs1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideSrcVT, N1: Rs1,
17393 N2: DAG.getUNDEF(VT: SrcVT));
17394 Rs2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideSrcVT, N1: Rs2,
17395 N2: DAG.getUNDEF(VT: SrcVT));
17396 SDValue Wide = DAG.getNode(Opcode: Opc, DL, VT: MVT::v2i32, N1: Rs1, N2: Rs2);
17397 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Wide, Idx: 0));
17398 return;
17399 }
17400
17401 case Intrinsic::riscv_pmul_00:
17402 case Intrinsic::riscv_pmul_01:
17403 case Intrinsic::riscv_pmul_11:
17404 case Intrinsic::riscv_pmulu_00:
17405 case Intrinsic::riscv_pmulu_01:
17406 case Intrinsic::riscv_pmulu_11:
17407 case Intrinsic::riscv_pmulsu_00:
17408 case Intrinsic::riscv_pmulsu_11: {
17409 MVT VT = N->getSimpleValueType(ResNo: 0);
17410 if (!Subtarget.is64Bit() || VT != MVT::v2i16)
17411 return;
17412
17413 SDValue Undef = DAG.getUNDEF(VT: MVT::v4i8);
17414 SDValue Rs1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i8,
17415 N1: N->getOperand(Num: 1), N2: Undef);
17416 SDValue Rs2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i8,
17417 N1: N->getOperand(Num: 2), N2: Undef);
17418 SDValue Res =
17419 DAG.getNode(Opcode: getRVPMulHalvesOpcode(IntNo), DL, VT: MVT::v4i16, N1: Rs1, N2: Rs2);
17420 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17421 return;
17422 }
17423 case Intrinsic::riscv_mul_00:
17424 case Intrinsic::riscv_mul_01:
17425 case Intrinsic::riscv_mul_11:
17426 case Intrinsic::riscv_mulu_00:
17427 case Intrinsic::riscv_mulu_01:
17428 case Intrinsic::riscv_mulu_11:
17429 case Intrinsic::riscv_mulsu_00:
17430 case Intrinsic::riscv_mulsu_11: {
17431 // mul.hXX exists only on RV32 and mul.wXX only on RV64; the other XLEN
17432 // has to build the product here.
17433 MVT VT = N->getSimpleValueType(ResNo: 0);
17434 MVT SrcVT = N->getOperand(Num: 1).getSimpleValueType();
17435 if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::i32 &&
17436 SrcVT == MVT::v2i16) {
17437 // The halfword product is the first element of the packed one.
17438 SDValue Undef = DAG.getUNDEF(VT: SrcVT);
17439 SDValue Rs1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16,
17440 N1: N->getOperand(Num: 1), N2: Undef);
17441 SDValue Rs2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16,
17442 N1: N->getOperand(Num: 2), N2: Undef);
17443 SDValue Res =
17444 DAG.getNode(Opcode: getRVPMulHalvesOpcode(IntNo), DL, VT: MVT::v2i32, N1: Rs1, N2: Rs2);
17445 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Res, Idx: 0));
17446 return;
17447 }
17448 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::i64 &&
17449 SrcVT == MVT::v2i32) {
17450 auto [Opc, Rs1Lane, Rs2Lane] = getRVPWordMulPartsOpcodeAndLanes(IntNo);
17451 SDValue Rs1 =
17452 DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: N->getOperand(Num: 1), Idx: Rs1Lane);
17453 SDValue Rs2 =
17454 DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: N->getOperand(Num: 2), Idx: Rs2Lane);
17455 SDValue Res =
17456 DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Rs1, N2: Rs2);
17457 Results.push_back(
17458 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Res, N2: Res.getValue(R: 1)));
17459 return;
17460 }
17461 reportFatalUsageError(reason: "unsupported llvm.riscv multiply-parts intrinsic");
17462 }
17463 case Intrinsic::riscv_macc_00:
17464 case Intrinsic::riscv_macc_01:
17465 case Intrinsic::riscv_macc_11:
17466 case Intrinsic::riscv_maccu_00:
17467 case Intrinsic::riscv_maccu_01:
17468 case Intrinsic::riscv_maccu_11:
17469 case Intrinsic::riscv_maccsu_00:
17470 case Intrinsic::riscv_maccsu_11: {
17471 // macc.hXX exists only on RV32 and macc.wXX only on RV64; the other XLEN
17472 // has to build the product here.
17473 MVT VT = N->getSimpleValueType(ResNo: 0);
17474 MVT SrcVT = N->getOperand(Num: 2).getSimpleValueType();
17475 if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::i32 &&
17476 SrcVT == MVT::v2i16) {
17477 // Accumulate into the first element of the packed product.
17478 SDValue Undef = DAG.getUNDEF(VT: SrcVT);
17479 SDValue Rd = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
17480 Operand: N->getOperand(Num: 1));
17481 SDValue Rs1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16,
17482 N1: N->getOperand(Num: 2), N2: Undef);
17483 SDValue Rs2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16,
17484 N1: N->getOperand(Num: 3), N2: Undef);
17485 SDValue Res = DAG.getNode(Opcode: getRVPMulAccHalvesOpcode(IntNo), DL,
17486 VT: MVT::v2i32, N1: Rd, N2: Rs1, N3: Rs2);
17487 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Res, Idx: 0));
17488 return;
17489 }
17490 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::i64 &&
17491 SrcVT == MVT::v2i32) {
17492 auto [Opc, Rs1Lane, Rs2Lane] =
17493 getRVPWordMulPartsAccOpcodeAndLanes(IntNo);
17494 auto [RdLo, RdHi] =
17495 DAG.SplitScalar(N: N->getOperand(Num: 1), DL, LoVT: MVT::i32, HiVT: MVT::i32);
17496 SDValue Rs1 =
17497 DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: N->getOperand(Num: 2), Idx: Rs1Lane);
17498 SDValue Rs2 =
17499 DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: N->getOperand(Num: 3), Idx: Rs2Lane);
17500 SDValue Res = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
17501 N1: RdLo, N2: RdHi, N3: Rs1, N4: Rs2);
17502 Results.push_back(
17503 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Res, N2: Res.getValue(R: 1)));
17504 return;
17505 }
17506 reportFatalUsageError(reason: "unsupported llvm.riscv multiply-parts intrinsic");
17507 }
17508 case Intrinsic::riscv_paadd:
17509 case Intrinsic::riscv_paaddu:
17510 case Intrinsic::riscv_pasub:
17511 case Intrinsic::riscv_pasubu:
17512 case Intrinsic::riscv_pabd:
17513 case Intrinsic::riscv_pabdu:
17514 case Intrinsic::riscv_pas:
17515 case Intrinsic::riscv_psa:
17516 case Intrinsic::riscv_psas:
17517 case Intrinsic::riscv_pssa:
17518 case Intrinsic::riscv_paas:
17519 case Intrinsic::riscv_pasa:
17520 case Intrinsic::riscv_pmerge:
17521 case Intrinsic::riscv_pmulq:
17522 case Intrinsic::riscv_pmulqr:
17523 case Intrinsic::riscv_pmulh:
17524 case Intrinsic::riscv_pmulhr:
17525 case Intrinsic::riscv_pmulhu:
17526 case Intrinsic::riscv_pmulhru:
17527 case Intrinsic::riscv_pmulhsu:
17528 case Intrinsic::riscv_pmulhrsu:
17529 case Intrinsic::riscv_psabs: {
17530 EVT VT = N->getValueType(ResNo: 0);
17531 if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
17532 return;
17533
17534 unsigned Opc;
17535 switch (IntNo) {
17536 case Intrinsic::riscv_paadd:
17537 Opc = ISD::AVGFLOORS;
17538 break;
17539 case Intrinsic::riscv_paaddu:
17540 Opc = ISD::AVGFLOORU;
17541 break;
17542 case Intrinsic::riscv_pasub:
17543 Opc = RISCVISD::ASUB;
17544 break;
17545 case Intrinsic::riscv_pasubu:
17546 Opc = RISCVISD::ASUBU;
17547 break;
17548 case Intrinsic::riscv_pabd:
17549 Opc = ISD::ABDS;
17550 break;
17551 case Intrinsic::riscv_pabdu:
17552 Opc = ISD::ABDU;
17553 break;
17554 case Intrinsic::riscv_psabs:
17555 Opc = RISCVISD::PSABS;
17556 break;
17557 case Intrinsic::riscv_pmulq:
17558 Opc = RISCVISD::MULQ;
17559 break;
17560 case Intrinsic::riscv_pmulqr:
17561 Opc = RISCVISD::MULQR;
17562 break;
17563 case Intrinsic::riscv_pmulh:
17564 case Intrinsic::riscv_pmulhr:
17565 case Intrinsic::riscv_pmulhu:
17566 case Intrinsic::riscv_pmulhru:
17567 case Intrinsic::riscv_pmulhsu:
17568 case Intrinsic::riscv_pmulhrsu:
17569 Opc = getRVPMulHighOpcode(IntNo);
17570 break;
17571 default:
17572 // pas/psa/psas/pssa/paas/pasa and pmerge: re-emit at the widened type
17573 // rather than lowering to a generic node.
17574 Opc = ISD::INTRINSIC_WO_CHAIN;
17575 break;
17576 }
17577
17578 EVT WideVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
17579 SDValue Undef = DAG.getUNDEF(VT);
17580 SmallVector<SDValue, 4> Ops(N->ops());
17581 for (SDValue &Op : Ops) {
17582 if (Op.getValueType() == VT)
17583 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT, N1: Op, N2: Undef);
17584 }
17585 SDValue Res;
17586 if (Opc == ISD::INTRINSIC_WO_CHAIN)
17587 Res = DAG.getNode(Opcode: Opc, DL, VT: WideVT, Ops);
17588 else
17589 Res = DAG.getNode(Opcode: Opc, DL, VT: WideVT, Ops: ArrayRef(Ops).slice(N: 1));
17590 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17591 return;
17592 }
17593 case Intrinsic::riscv_pnclipp:
17594 case Intrinsic::riscv_pnclipup: {
17595 bool IsSigned = IntNo == Intrinsic::riscv_pnclipp;
17596 EVT VT = N->getValueType(ResNo: 0);
17597 if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
17598 return;
17599 unsigned Opc = IsSigned ? RISCVISD::PNCLIPP : RISCVISD::PNCLIPUP;
17600 SDValue Src1 = N->getOperand(Num: 1);
17601 SDValue Src2 = N->getOperand(Num: 2);
17602 if (VT == MVT::v4i8) {
17603 MVT WideSrcVT = MVT::v4i16;
17604 SDValue Packed =
17605 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideSrcVT, Ops: {Src1, Src2});
17606 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::v8i8, Ops: {Packed, Packed});
17607 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17608 } else {
17609 MVT WideSrcVT = MVT::v2i32;
17610 SDValue Packed =
17611 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: WideSrcVT, Ops: {Src1, Src2});
17612 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::v4i16, Ops: {Packed, Packed});
17613 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17614 }
17615 return;
17616 }
17617 case Intrinsic::riscv_pssha:
17618 case Intrinsic::riscv_psshar:
17619 case Intrinsic::riscv_psshl:
17620 case Intrinsic::riscv_psshlr: {
17621 MVT VT = N->getSimpleValueType(ResNo: 0);
17622 if (!Subtarget.is64Bit() || VT != MVT::v2i16)
17623 return;
17624
17625 MVT WideVT = MVT::v4i16;
17626 SDValue Op0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT,
17627 N1: N->getOperand(Num: 1), N2: DAG.getUNDEF(VT));
17628 SDValue ShAmt = N->getOperand(Num: 2);
17629 ShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: Subtarget.getXLenVT());
17630 SDValue Res =
17631 DAG.getNode(Opcode: getRVPShiftOpcode(IntNo), DL, VT: WideVT, N1: Op0, N2: ShAmt);
17632 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
17633 return;
17634 }
17635 case Intrinsic::riscv_predsum:
17636 case Intrinsic::riscv_predsumu: {
17637 bool IsSigned = IntNo == Intrinsic::riscv_predsum;
17638 SDValue Vec = N->getOperand(Num: 1);
17639 MVT VecVT = Vec.getSimpleValueType();
17640 auto Ext = [&](SDValue V) {
17641 return DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
17642 VT: MVT::i64, Operand: V);
17643 };
17644 auto RedSum = [&](MVT VT, SDValue V, SDValue Acc) {
17645 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: N->getOperand(Num: 0), N2: V,
17646 N3: Acc);
17647 };
17648
17649 // RV32: i64 accumulator. Reduce to a 32-bit partial sum, then
17650 // widening-accumulate into i64 via wadda/waddau (v2i32 uses wadda alone).
17651 if (!Subtarget.is64Bit() && N->getValueType(ResNo: 0) == MVT::i64) {
17652 SDValue Acc = N->getOperand(Num: 2);
17653 SDValue Res;
17654 if (VecVT == MVT::v2i32) {
17655 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Acc,
17656 N2: Ext(DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec, Idx: 0)));
17657 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Res,
17658 N2: Ext(DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec, Idx: 1)));
17659 } else {
17660 // The paired predsum.dbs/dhs computes the 32-bit element sum.
17661 SDValue Partial =
17662 RedSum(MVT::i32, Vec, DAG.getConstant(Val: 0, DL, VT: MVT::i32));
17663 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Acc, N2: Ext(Partial));
17664 }
17665 Results.push_back(Elt: Res);
17666 return;
17667 }
17668
17669 // RV64: i32 accumulator. Reduce at i64 (XLEN), then truncate.
17670 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
17671 return;
17672
17673 // Zero the upper lanes (zext.w) so they don't contribute to the sum.
17674 if (VecVT == MVT::v4i8 || VecVT == MVT::v2i16)
17675 Vec = widenPackedVectorWithZeros(
17676 DAG, DL, V: Vec, WideVT: VecVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16);
17677
17678 // The result is truncated to i32, so the accumulator's upper bits are
17679 // unused and need no sign/zero extension.
17680 SDValue Acc =
17681 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
17682 SDValue Res = RedSum(MVT::i64, Vec, Acc);
17683 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17684 return;
17685 }
17686 case Intrinsic::riscv_pabdsumu:
17687 case Intrinsic::riscv_pabdsumau: {
17688 bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
17689 // The two packed sources (rs1, rs2) are the last two operands.
17690 SDValue Rs1 = N->getOperand(Num: N->getNumOperands() - 2);
17691 SDValue Rs2 = N->getOperand(Num: N->getNumOperands() - 1);
17692 MVT VecVT = Rs1.getSimpleValueType();
17693
17694 // RV32: i64 result, always from a v8i8 source. The accumulator, if any,
17695 // folds into the widening add below.
17696 if (!Subtarget.is64Bit() && N->getValueType(ResNo: 0) == MVT::i64) {
17697 // Sum of absolute differences of two v4i8 halves.
17698 auto Sad = [&](SDValue A, SDValue B) {
17699 SDValue S = DAG.getNode(
17700 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
17701 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_pabdsumu, DL, VT: MVT::i32), N2: A,
17702 N3: B);
17703 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: S);
17704 };
17705 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
17706 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
17707 SDValue Lo = Sad(Rs1Lo, Rs2Lo);
17708 SDValue Hi = Sad(Rs1Hi, Rs2Hi);
17709 // (acc + lo) + hi keeps the accumulate chained so it folds into a
17710 // single waddau; without an accumulator lo + hi folds into waddu.
17711 SDValue Res =
17712 IsAcc ? DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: N->getOperand(Num: 1), N2: Lo)
17713 : Lo;
17714 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Res, N2: Hi);
17715 Results.push_back(Elt: Res);
17716 return;
17717 }
17718
17719 // RV64: i32 result, so reduce at i64 and truncate. The source is v4i8 or
17720 // v8i8; widen a v4i8 to v8i8, zeroing the upper bytes (v8i8 is legal).
17721 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
17722 return;
17723 if (VecVT == MVT::v4i8) {
17724 Rs1 = widenPackedVectorWithZeros(DAG, DL, V: Rs1, WideVT: MVT::v8i8);
17725 Rs2 = widenPackedVectorWithZeros(DAG, DL, V: Rs2, WideVT: MVT::v8i8);
17726 }
17727 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0)};
17728 if (IsAcc)
17729 Ops.push_back(
17730 Elt: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1)));
17731 Ops.push_back(Elt: Rs1);
17732 Ops.push_back(Elt: Rs2);
17733 SDValue Res = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i64, Ops);
17734 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17735 return;
17736 }
17737 case Intrinsic::riscv_mqacc_00:
17738 case Intrinsic::riscv_mqacc_01:
17739 case Intrinsic::riscv_mqacc_11:
17740 case Intrinsic::riscv_mqracc_00:
17741 case Intrinsic::riscv_mqracc_01:
17742 case Intrinsic::riscv_mqracc_11: {
17743 EVT VT = N->getValueType(ResNo: 0);
17744 if (Subtarget.is64Bit() && VT == MVT::i32) {
17745 SDValue Rd = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
17746 Operand: N->getOperand(Num: 1));
17747 auto WidenSrc = [&](SDValue V) {
17748 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v4i16,
17749 Ops: {V, DAG.getUNDEF(VT: MVT::v2i16)});
17750 };
17751 SDValue Rs1 = WidenSrc(N->getOperand(Num: 2));
17752 SDValue Rs2 = WidenSrc(N->getOperand(Num: 3));
17753 unsigned Opc = getRVPQFormatAccOpcode(IntNo);
17754 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::v2i32, N1: Rd, N2: Rs1, N3: Rs2);
17755 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: Res, Idx: 0));
17756 return;
17757 }
17758
17759 if (!Subtarget.is64Bit() && VT == MVT::i64) {
17760 bool IsRound = IntNo == Intrinsic::riscv_mqracc_00 ||
17761 IntNo == Intrinsic::riscv_mqracc_01 ||
17762 IntNo == Intrinsic::riscv_mqracc_11;
17763 bool Hi1 = IntNo == Intrinsic::riscv_mqacc_11 ||
17764 IntNo == Intrinsic::riscv_mqracc_11;
17765 bool Hi2 = IntNo == Intrinsic::riscv_mqacc_01 ||
17766 IntNo == Intrinsic::riscv_mqacc_11 ||
17767 IntNo == Intrinsic::riscv_mqracc_01 ||
17768 IntNo == Intrinsic::riscv_mqracc_11;
17769 MVT XLenVT = Subtarget.getXLenVT();
17770 SDValue Rs1 = N->getOperand(Num: 2);
17771 SDValue Rs2 = N->getOperand(Num: 3);
17772 SDValue A = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rs1, Idx: Hi1 ? 1 : 0);
17773 SDValue B = DAG.getExtractVectorElt(DL, VT: XLenVT, Vec: Rs2, Idx: Hi2 ? 1 : 0);
17774 auto [RdLo, RdHi] =
17775 DAG.SplitScalar(N: N->getOperand(Num: 1), DL, LoVT: XLenVT, HiVT: XLenVT);
17776 unsigned Opc = IsRound ? RISCVISD::MQRWACC : RISCVISD::MQWACC;
17777 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: XLenVT);
17778 SDValue Acc = DAG.getNode(Opcode: Opc, DL, VTList: VTs, Ops: {RdLo, RdHi, A, B});
17779 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64,
17780 N1: Acc.getValue(R: 0), N2: Acc.getValue(R: 1)));
17781 return;
17782 }
17783 return;
17784 }
17785 case Intrinsic::riscv_orc_b:
17786 case Intrinsic::riscv_brev8:
17787 case Intrinsic::riscv_sha256sig0:
17788 case Intrinsic::riscv_sha256sig1:
17789 case Intrinsic::riscv_sha256sum0:
17790 case Intrinsic::riscv_sha256sum1:
17791 case Intrinsic::riscv_sm3p0:
17792 case Intrinsic::riscv_sm3p1: {
17793 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
17794 return;
17795 unsigned Opc;
17796 switch (IntNo) {
17797 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
17798 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
17799 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
17800 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
17801 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
17802 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
17803 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
17804 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
17805 }
17806
17807 SDValue NewOp =
17808 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17809 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, Operand: NewOp);
17810 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17811 return;
17812 }
17813 case Intrinsic::riscv_sm4ks:
17814 case Intrinsic::riscv_sm4ed: {
17815 unsigned Opc =
17816 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
17817 SDValue NewOp0 =
17818 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17819 SDValue NewOp1 =
17820 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
17821 SDValue Res =
17822 DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1, N3: N->getOperand(Num: 3));
17823 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17824 return;
17825 }
17826 case Intrinsic::riscv_mopr: {
17827 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
17828 return;
17829 SDValue NewOp =
17830 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17831 SDValue Res = DAG.getNode(
17832 Opcode: RISCVISD::MOP_R, DL, VT: MVT::i64, N1: NewOp,
17833 N2: DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 2), DL, VT: MVT::i64));
17834 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17835 return;
17836 }
17837 case Intrinsic::riscv_moprr: {
17838 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
17839 return;
17840 SDValue NewOp0 =
17841 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17842 SDValue NewOp1 =
17843 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
17844 SDValue Res = DAG.getNode(
17845 Opcode: RISCVISD::MOP_RR, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1,
17846 N3: DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 3), DL, VT: MVT::i64));
17847 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17848 return;
17849 }
17850 case Intrinsic::riscv_clmulh:
17851 case Intrinsic::riscv_clmulr: {
17852 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
17853 return;
17854
17855 // Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
17856 // to the full 128-bit clmul result of multiplying two xlen values.
17857 // Perform clmulr or clmulh on the shifted values. Finally, extract the
17858 // upper 32 bits.
17859 //
17860 // The alternative is to mask the inputs to 32 bits and use clmul, but
17861 // that requires two shifts to mask each input without zext.w.
17862 // FIXME: If the inputs are known zero extended or could be freely
17863 // zero extended, the mask form would be better.
17864 SDValue NewOp0 =
17865 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
17866 SDValue NewOp1 =
17867 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
17868 NewOp0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp0,
17869 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
17870 NewOp1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp1,
17871 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
17872 unsigned Opc =
17873 IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
17874 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
17875 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Res,
17876 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
17877 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
17878 return;
17879 }
17880 case Intrinsic::riscv_vmv_x_s: {
17881 EVT VT = N->getValueType(ResNo: 0);
17882 MVT XLenVT = Subtarget.getXLenVT();
17883 if (VT.bitsLT(VT: XLenVT)) {
17884 // Simple case just extract using vmv.x.s and truncate.
17885 SDValue Extract = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL,
17886 VT: Subtarget.getXLenVT(), Operand: N->getOperand(Num: 1));
17887 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Extract));
17888 return;
17889 }
17890
17891 assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
17892 "Unexpected custom legalization");
17893
17894 // We need to do the move in two steps.
17895 SDValue Vec = N->getOperand(Num: 1);
17896 MVT VecVT = Vec.getSimpleValueType();
17897
17898 // First extract the lower XLEN bits of the element.
17899 SDValue EltLo = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
17900
17901 // To extract the upper XLEN bits of the vector element, shift the first
17902 // element right by 32 bits and re-extract the lower XLEN bits.
17903 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT: VecVT, DL, DAG, Subtarget);
17904
17905 SDValue ThirtyTwoV =
17906 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: VecVT, N1: DAG.getUNDEF(VT: VecVT),
17907 N2: DAG.getConstant(Val: 32, DL, VT: XLenVT), N3: VL);
17908 SDValue LShr32 = DAG.getNode(Opcode: RISCVISD::SRL_VL, DL, VT: VecVT, N1: Vec, N2: ThirtyTwoV,
17909 N3: DAG.getUNDEF(VT: VecVT), N4: Mask, N5: VL);
17910 SDValue EltHi = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: LShr32);
17911
17912 Results.push_back(
17913 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: EltLo, N2: EltHi));
17914 break;
17915 }
17916 }
17917 break;
17918 }
17919 case ISD::VECREDUCE_ADD:
17920 case ISD::VECREDUCE_AND:
17921 case ISD::VECREDUCE_OR:
17922 case ISD::VECREDUCE_XOR:
17923 case ISD::VECREDUCE_SMAX:
17924 case ISD::VECREDUCE_UMAX:
17925 case ISD::VECREDUCE_SMIN:
17926 case ISD::VECREDUCE_UMIN:
17927 if (SDValue V = lowerVECREDUCE(Op: SDValue(N, 0), DAG))
17928 Results.push_back(Elt: V);
17929 break;
17930 case ISD::VP_REDUCE_ADD:
17931 case ISD::VP_REDUCE_AND:
17932 case ISD::VP_REDUCE_OR:
17933 case ISD::VP_REDUCE_XOR:
17934 case ISD::VP_REDUCE_SMAX:
17935 case ISD::VP_REDUCE_UMAX:
17936 case ISD::VP_REDUCE_SMIN:
17937 case ISD::VP_REDUCE_UMIN:
17938 if (SDValue V = lowerVPREDUCE(Op: SDValue(N, 0), DAG))
17939 Results.push_back(Elt: V);
17940 break;
17941 case ISD::GET_ROUNDING: {
17942 SDVTList VTs = DAG.getVTList(VT1: Subtarget.getXLenVT(), VT2: MVT::Other);
17943 SDValue Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL, VTList: VTs, N: N->getOperand(Num: 0));
17944 Results.push_back(Elt: Res.getValue(R: 0));
17945 Results.push_back(Elt: Res.getValue(R: 1));
17946 break;
17947 }
17948 }
17949}
17950
17951/// Given a binary operator, return the *associative* generic ISD::VECREDUCE_OP
17952/// which corresponds to it.
17953static unsigned getVecReduceOpcode(unsigned Opc) {
17954 switch (Opc) {
17955 default:
17956 llvm_unreachable("Unhandled binary to transform reduction");
17957 case ISD::ADD:
17958 return ISD::VECREDUCE_ADD;
17959 case ISD::UMAX:
17960 return ISD::VECREDUCE_UMAX;
17961 case ISD::SMAX:
17962 return ISD::VECREDUCE_SMAX;
17963 case ISD::UMIN:
17964 return ISD::VECREDUCE_UMIN;
17965 case ISD::SMIN:
17966 return ISD::VECREDUCE_SMIN;
17967 case ISD::AND:
17968 return ISD::VECREDUCE_AND;
17969 case ISD::OR:
17970 return ISD::VECREDUCE_OR;
17971 case ISD::XOR:
17972 return ISD::VECREDUCE_XOR;
17973 case ISD::FADD:
17974 // Note: This is the associative form of the generic reduction opcode.
17975 return ISD::VECREDUCE_FADD;
17976 case ISD::FMAXNUM:
17977 return ISD::VECREDUCE_FMAX;
17978 case ISD::FMINNUM:
17979 return ISD::VECREDUCE_FMIN;
17980 }
17981}
17982
17983/// Perform two related transforms whose purpose is to incrementally recognize
17984/// an explode_vector followed by scalar reduction as a vector reduction node.
17985/// This exists to recover from a deficiency in SLP which can't handle
17986/// forests with multiple roots sharing common nodes. In some cases, one
17987/// of the trees will be vectorized, and the other will remain (unprofitably)
17988/// scalarized.
17989static SDValue
17990combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
17991 const RISCVSubtarget &Subtarget) {
17992
17993 // This transforms need to run before all integer types have been legalized
17994 // to i64 (so that the vector element type matches the add type), and while
17995 // it's safe to introduce odd sized vector types.
17996 if (DAG.NewNodesMustHaveLegalTypes)
17997 return SDValue();
17998
17999 // Without V, this transform isn't useful. We could form the (illegal)
18000 // operations and let them be scalarized again, but there's really no point.
18001 if (!Subtarget.hasVInstructions())
18002 return SDValue();
18003
18004 const SDLoc DL(N);
18005 const EVT VT = N->getValueType(ResNo: 0);
18006 const unsigned Opc = N->getOpcode();
18007
18008 if (!VT.isInteger()) {
18009 switch (Opc) {
18010 default:
18011 return SDValue();
18012 case ISD::FADD:
18013 // For FADD, we only handle the case with reassociation allowed. We
18014 // could handle strict reduction order, but at the moment, there's no
18015 // known reason to, and the complexity isn't worth it.
18016 if (!N->getFlags().hasAllowReassociation())
18017 return SDValue();
18018 break;
18019 case ISD::FMAXNUM:
18020 case ISD::FMINNUM:
18021 break;
18022 }
18023 }
18024
18025 const unsigned ReduceOpc = getVecReduceOpcode(Opc);
18026 assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
18027 "Inconsistent mappings");
18028 SDValue LHS = N->getOperand(Num: 0);
18029 SDValue RHS = N->getOperand(Num: 1);
18030
18031 if (!LHS.hasOneUse() || !RHS.hasOneUse())
18032 return SDValue();
18033
18034 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
18035 std::swap(a&: LHS, b&: RHS);
18036
18037 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18038 !isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)))
18039 return SDValue();
18040
18041 uint64_t RHSIdx = cast<ConstantSDNode>(Val: RHS.getOperand(i: 1))->getLimitedValue();
18042 SDValue SrcVec = RHS.getOperand(i: 0);
18043 EVT SrcVecVT = SrcVec.getValueType();
18044 if (SrcVecVT.getVectorElementType() != VT)
18045 return SDValue();
18046 if (SrcVecVT.isScalableVector())
18047 return SDValue();
18048
18049 if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen())
18050 return SDValue();
18051
18052 // match binop (extract_vector_elt V, 0), (extract_vector_elt V, 1) to
18053 // reduce_op (extract_subvector [2 x VT] from V). This will form the
18054 // root of our reduction tree. TODO: We could extend this to any two
18055 // adjacent aligned constant indices if desired.
18056 if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
18057 LHS.getOperand(i: 0) == SrcVec && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
18058 uint64_t LHSIdx =
18059 cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))->getLimitedValue();
18060 if (0 == std::min(a: LHSIdx, b: RHSIdx) && 1 == std::max(a: LHSIdx, b: RHSIdx)) {
18061 EVT ReduceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 2);
18062 SDValue Vec = DAG.getExtractSubvector(DL, VT: ReduceVT, Vec: SrcVec, Idx: 0);
18063 return DAG.getNode(Opcode: ReduceOpc, DL, VT, Operand: Vec, Flags: N->getFlags());
18064 }
18065 }
18066
18067 // Match (binop (reduce (extract_subvector V, 0),
18068 // (extract_vector_elt V, sizeof(SubVec))))
18069 // into a reduction of one more element from the original vector V.
18070 if (LHS.getOpcode() != ReduceOpc)
18071 return SDValue();
18072
18073 SDValue ReduceVec = LHS.getOperand(i: 0);
18074 if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
18075 ReduceVec.hasOneUse() && ReduceVec.getOperand(i: 0) == RHS.getOperand(i: 0) &&
18076 isNullConstant(V: ReduceVec.getOperand(i: 1)) &&
18077 ReduceVec.getValueType().getVectorNumElements() == RHSIdx) {
18078 // For illegal types (e.g. 3xi32), most will be combined again into a
18079 // wider (hopefully legal) type. If this is a terminal state, we are
18080 // relying on type legalization here to produce something reasonable
18081 // and this lowering quality could probably be improved. (TODO)
18082 EVT ReduceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: RHSIdx + 1);
18083 SDValue Vec = DAG.getExtractSubvector(DL, VT: ReduceVT, Vec: SrcVec, Idx: 0);
18084 return DAG.getNode(Opcode: ReduceOpc, DL, VT, Operand: Vec,
18085 Flags: ReduceVec->getFlags() & N->getFlags());
18086 }
18087
18088 return SDValue();
18089}
18090
18091
18092// Try to fold (<bop> x, (reduction.<bop> vec, start))
18093static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
18094 const RISCVSubtarget &Subtarget) {
18095 auto BinOpToRVVReduce = [](unsigned Opc) {
18096 switch (Opc) {
18097 default:
18098 llvm_unreachable("Unhandled binary to transform reduction");
18099 case ISD::ADD:
18100 return RISCVISD::VECREDUCE_ADD_VL;
18101 case ISD::UMAX:
18102 return RISCVISD::VECREDUCE_UMAX_VL;
18103 case ISD::SMAX:
18104 return RISCVISD::VECREDUCE_SMAX_VL;
18105 case ISD::UMIN:
18106 return RISCVISD::VECREDUCE_UMIN_VL;
18107 case ISD::SMIN:
18108 return RISCVISD::VECREDUCE_SMIN_VL;
18109 case ISD::AND:
18110 return RISCVISD::VECREDUCE_AND_VL;
18111 case ISD::OR:
18112 return RISCVISD::VECREDUCE_OR_VL;
18113 case ISD::XOR:
18114 return RISCVISD::VECREDUCE_XOR_VL;
18115 case ISD::FADD:
18116 return RISCVISD::VECREDUCE_FADD_VL;
18117 case ISD::FMAXNUM:
18118 return RISCVISD::VECREDUCE_FMAX_VL;
18119 case ISD::FMINNUM:
18120 return RISCVISD::VECREDUCE_FMIN_VL;
18121 }
18122 };
18123
18124 auto IsReduction = [&BinOpToRVVReduce](SDValue V, unsigned Opc) {
18125 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
18126 isNullConstant(V: V.getOperand(i: 1)) &&
18127 V.getOperand(i: 0).getOpcode() == BinOpToRVVReduce(Opc);
18128 };
18129
18130 unsigned Opc = N->getOpcode();
18131 unsigned ReduceIdx;
18132 if (IsReduction(N->getOperand(Num: 0), Opc))
18133 ReduceIdx = 0;
18134 else if (IsReduction(N->getOperand(Num: 1), Opc))
18135 ReduceIdx = 1;
18136 else
18137 return SDValue();
18138
18139 // Skip if FADD disallows reassociation but the combiner needs.
18140 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
18141 return SDValue();
18142
18143 SDValue Extract = N->getOperand(Num: ReduceIdx);
18144 SDValue Reduce = Extract.getOperand(i: 0);
18145 if (!Extract.hasOneUse() || !Reduce.hasOneUse())
18146 return SDValue();
18147
18148 SDValue ScalarV = Reduce.getOperand(i: 2);
18149 EVT ScalarVT = ScalarV.getValueType();
18150 if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
18151 ScalarV.getOperand(i: 0)->isUndef() &&
18152 isNullConstant(V: ScalarV.getOperand(i: 2)))
18153 ScalarV = ScalarV.getOperand(i: 1);
18154
18155 // Make sure that ScalarV is a splat with VL=1.
18156 if (ScalarV.getOpcode() != RISCVISD::VFMV_S_F_VL &&
18157 ScalarV.getOpcode() != RISCVISD::VMV_S_X_VL &&
18158 ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
18159 return SDValue();
18160
18161 if (!isNonZeroAVL(AVL: ScalarV.getOperand(i: 2)))
18162 return SDValue();
18163
18164 // Check the scalar of ScalarV is neutral element
18165 // TODO: Deal with value other than neutral element.
18166 if (!DAG.isIdentityElement(Opc: N->getOpcode(), Flags: N->getFlags(),
18167 V: ScalarV.getOperand(i: 1), OperandNo: 0))
18168 return SDValue();
18169
18170 // If the AVL is zero, operand 0 will be returned. So it's not safe to fold.
18171 // FIXME: We might be able to improve this if operand 0 is undef.
18172 if (!isNonZeroAVL(AVL: Reduce.getOperand(i: 5)))
18173 return SDValue();
18174
18175 SDValue NewStart = N->getOperand(Num: 1 - ReduceIdx);
18176
18177 SDLoc DL(N);
18178 SDValue NewScalarV =
18179 lowerScalarInsert(Scalar: NewStart, VL: ScalarV.getOperand(i: 2),
18180 VT: ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
18181
18182 // If we looked through an INSERT_SUBVECTOR we need to restore it.
18183 if (ScalarVT != ScalarV.getValueType())
18184 NewScalarV =
18185 DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ScalarVT), SubVec: NewScalarV, Idx: 0);
18186
18187 SDValue Ops[] = {Reduce.getOperand(i: 0), Reduce.getOperand(i: 1),
18188 NewScalarV, Reduce.getOperand(i: 3),
18189 Reduce.getOperand(i: 4), Reduce.getOperand(i: 5)};
18190 SDValue NewReduce =
18191 DAG.getNode(Opcode: Reduce.getOpcode(), DL, VT: Reduce.getValueType(), Ops);
18192 return DAG.getNode(Opcode: Extract.getOpcode(), DL, VT: Extract.getValueType(), N1: NewReduce,
18193 N2: Extract.getOperand(i: 1));
18194}
18195
18196// Optimize (add (shl x, c0), (shl y, c1)) ->
18197// (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
18198// or
18199// (SLLI (QC.SHLADD x, y, c1 - c0), c0), if 4 <= (c1-c0) <=31.
18200static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
18201 const RISCVSubtarget &Subtarget) {
18202 // Perform this optimization only in the zba/xandesperf/xqciac/xtheadba
18203 // extension.
18204 if (!Subtarget.hasShlAdd(ShAmt: 3))
18205 return SDValue();
18206
18207 // Skip for vector types and larger types.
18208 EVT VT = N->getValueType(ResNo: 0);
18209 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
18210 return SDValue();
18211
18212 // The two operand nodes must be SHL and have no other use.
18213 SDValue N0 = N->getOperand(Num: 0);
18214 SDValue N1 = N->getOperand(Num: 1);
18215 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
18216 !N0->hasOneUse() || !N1->hasOneUse())
18217 return SDValue();
18218
18219 // Check c0 and c1.
18220 auto *N0C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
18221 auto *N1C = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1));
18222 if (!N0C || !N1C)
18223 return SDValue();
18224 int64_t C0 = N0C->getSExtValue();
18225 int64_t C1 = N1C->getSExtValue();
18226 if (C0 <= 0 || C1 <= 0)
18227 return SDValue();
18228
18229 int64_t Diff = std::abs(i: C0 - C1);
18230 if (!Subtarget.hasShlAdd(ShAmt: Diff))
18231 return SDValue();
18232
18233 // Build nodes.
18234 SDLoc DL(N);
18235 int64_t Bits = std::min(a: C0, b: C1);
18236 SDValue NS = (C0 < C1) ? N0->getOperand(Num: 0) : N1->getOperand(Num: 0);
18237 SDValue NL = (C0 > C1) ? N0->getOperand(Num: 0) : N1->getOperand(Num: 0);
18238 SDValue SHADD = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: NL,
18239 N2: DAG.getTargetConstant(Val: Diff, DL, VT), N3: NS);
18240 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: SHADD, N2: DAG.getConstant(Val: Bits, DL, VT));
18241}
18242
18243// Check if this SDValue is an add immediate that is fed by a shift of 1, 2,
18244// or 3.
18245static SDValue combineShlAddIAddImpl(SDNode *N, SDValue AddI, SDValue Other,
18246 SelectionDAG &DAG) {
18247 using namespace llvm::SDPatternMatch;
18248
18249 // Looking for a reg-reg add and not an addi.
18250 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
18251 return SDValue();
18252
18253 // Based on testing it seems that performance degrades if the ADDI has
18254 // more than 2 uses.
18255 if (AddI->use_size() > 2)
18256 return SDValue();
18257
18258 APInt AddVal;
18259 SDValue SHLVal;
18260 if (!sd_match(N: AddI, P: m_Add(L: m_Value(N&: SHLVal), R: m_ConstInt(V&: AddVal))))
18261 return SDValue();
18262
18263 APInt VShift;
18264 if (!sd_match(N: SHLVal, P: m_OneUse(P: m_Shl(L: m_Value(), R: m_ConstInt(V&: VShift)))))
18265 return SDValue();
18266
18267 if (VShift.slt(RHS: 1) || VShift.sgt(RHS: 3))
18268 return SDValue();
18269
18270 SDLoc DL(N);
18271 EVT VT = N->getValueType(ResNo: 0);
18272 // The shift must be positive but the add can be signed.
18273 uint64_t ShlConst = VShift.getZExtValue();
18274 int64_t AddConst = AddVal.getSExtValue();
18275
18276 SDValue SHADD = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: SHLVal->getOperand(Num: 0),
18277 N2: DAG.getTargetConstant(Val: ShlConst, DL, VT), N3: Other);
18278 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: SHADD,
18279 N2: DAG.getSignedConstant(Val: AddConst, DL, VT));
18280}
18281
18282// Optimize (add (add (shl x, c0), c1), y) ->
18283// (ADDI (SH*ADD y, x), c1), if c0 equals to [1|2|3].
18284static SDValue combineShlAddIAdd(SDNode *N, SelectionDAG &DAG,
18285 const RISCVSubtarget &Subtarget) {
18286 // Perform this optimization only in the zba extension.
18287 if (!ReassocShlAddiAdd || !Subtarget.hasShlAdd(ShAmt: 3))
18288 return SDValue();
18289
18290 // Skip for vector types and larger types.
18291 EVT VT = N->getValueType(ResNo: 0);
18292 if (VT != Subtarget.getXLenVT())
18293 return SDValue();
18294
18295 SDValue AddI = N->getOperand(Num: 0);
18296 SDValue Other = N->getOperand(Num: 1);
18297 if (SDValue V = combineShlAddIAddImpl(N, AddI, Other, DAG))
18298 return V;
18299 if (SDValue V = combineShlAddIAddImpl(N, AddI: Other, Other: AddI, DAG))
18300 return V;
18301 return SDValue();
18302}
18303
18304// Combine a constant select operand into its use:
18305//
18306// (and (select cond, -1, c), x)
18307// -> (select cond, x, (and x, c)) [AllOnes=1]
18308// (or (select cond, 0, c), x)
18309// -> (select cond, x, (or x, c)) [AllOnes=0]
18310// (xor (select cond, 0, c), x)
18311// -> (select cond, x, (xor x, c)) [AllOnes=0]
18312// (add (select cond, 0, c), x)
18313// -> (select cond, x, (add x, c)) [AllOnes=0]
18314// (sub x, (select cond, 0, c))
18315// -> (select cond, x, (sub x, c)) [AllOnes=0]
18316static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
18317 SelectionDAG &DAG, bool AllOnes,
18318 const RISCVSubtarget &Subtarget) {
18319 EVT VT = N->getValueType(ResNo: 0);
18320
18321 // Skip vectors.
18322 if (VT.isVector())
18323 return SDValue();
18324
18325 if (!Subtarget.hasConditionalMoveFusion()) {
18326 // (select cond, x, (and x, c)) has custom lowering with Zicond.
18327 if (!Subtarget.hasStdExtZicond() || N->getOpcode() != ISD::AND)
18328 return SDValue();
18329
18330 // Maybe harmful when condition code has multiple use.
18331 if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(i: 0).hasOneUse())
18332 return SDValue();
18333
18334 // Maybe harmful when VT is wider than XLen.
18335 if (VT.getSizeInBits() > Subtarget.getXLen())
18336 return SDValue();
18337 }
18338
18339 if ((Slct.getOpcode() != ISD::SELECT &&
18340 Slct.getOpcode() != RISCVISD::SELECT_CC) ||
18341 !Slct.hasOneUse())
18342 return SDValue();
18343
18344 auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
18345 return AllOnes ? isAllOnesConstant(V: N) : isNullConstant(V: N);
18346 };
18347
18348 bool SwapSelectOps;
18349 unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
18350 SDValue TrueVal = Slct.getOperand(i: 1 + OpOffset);
18351 SDValue FalseVal = Slct.getOperand(i: 2 + OpOffset);
18352 SDValue NonConstantVal;
18353 if (isZeroOrAllOnes(TrueVal, AllOnes)) {
18354 SwapSelectOps = false;
18355 NonConstantVal = FalseVal;
18356 } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
18357 SwapSelectOps = true;
18358 NonConstantVal = TrueVal;
18359 } else
18360 return SDValue();
18361
18362 // Slct is now know to be the desired identity constant when CC is true.
18363 TrueVal = OtherOp;
18364 FalseVal = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, N1: OtherOp, N2: NonConstantVal);
18365 // Unless SwapSelectOps says the condition should be false.
18366 if (SwapSelectOps)
18367 std::swap(a&: TrueVal, b&: FalseVal);
18368
18369 if (Slct.getOpcode() == RISCVISD::SELECT_CC)
18370 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL: SDLoc(N), VT,
18371 Ops: {Slct.getOperand(i: 0), Slct.getOperand(i: 1),
18372 Slct.getOperand(i: 2), TrueVal, FalseVal});
18373
18374 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT,
18375 Ops: {Slct.getOperand(i: 0), TrueVal, FalseVal});
18376}
18377
18378// Attempt combineSelectAndUse on each operand of a commutative operator N.
18379static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
18380 bool AllOnes,
18381 const RISCVSubtarget &Subtarget) {
18382 SDValue N0 = N->getOperand(Num: 0);
18383 SDValue N1 = N->getOperand(Num: 1);
18384 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DAG, AllOnes, Subtarget))
18385 return Result;
18386 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DAG, AllOnes, Subtarget))
18387 return Result;
18388 return SDValue();
18389}
18390
18391// Transform (add (mul x, c0), c1) ->
18392// (add (mul (add x, c1/c0), c0), c1%c0).
18393// if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
18394// that should be excluded is when c0*(c1/c0) is simm12, which will lead
18395// to an infinite loop in DAGCombine if transformed.
18396// Or transform (add (mul x, c0), c1) ->
18397// (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
18398// if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
18399// case that should be excluded is when c0*(c1/c0+1) is simm12, which will
18400// lead to an infinite loop in DAGCombine if transformed.
18401// Or transform (add (mul x, c0), c1) ->
18402// (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
18403// if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
18404// case that should be excluded is when c0*(c1/c0-1) is simm12, which will
18405// lead to an infinite loop in DAGCombine if transformed.
18406// Or transform (add (mul x, c0), c1) ->
18407// (mul (add x, c1/c0), c0).
18408// if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
18409static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
18410 const RISCVSubtarget &Subtarget) {
18411 // Skip for vector types and larger types.
18412 EVT VT = N->getValueType(ResNo: 0);
18413 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
18414 return SDValue();
18415 // The first operand node must be a MUL and has no other use.
18416 SDValue N0 = N->getOperand(Num: 0);
18417 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
18418 return SDValue();
18419 // Check if c0 and c1 match above conditions.
18420 auto *N0C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
18421 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
18422 if (!N0C || !N1C)
18423 return SDValue();
18424 // If N0C has multiple uses it's possible one of the cases in
18425 // DAGCombiner::isMulAddWithConstProfitable will be true, which would result
18426 // in an infinite loop.
18427 if (!N0C->hasOneUse())
18428 return SDValue();
18429 int64_t C0 = N0C->getSExtValue();
18430 int64_t C1 = N1C->getSExtValue();
18431 int64_t CA, CB;
18432 // If C1 already fits in an add immediate, there is nothing to split out: the
18433 // (add (mul x, c0), c1) form is already canonical/cheap. Splitting it would
18434 // fight the generic DAGCombiner fold add(mul(add(A, CA), CM), CB) ->
18435 // add(mul(A, CM), CM*CA+CB) (which is gated on isLegalAddImmediate) and cause
18436 // an infinite loop.
18437 if (C0 == -1 || C0 == 0 || C0 == 1 ||
18438 Subtarget.getTargetLowering()->isLegalAddImmediate(Imm: C1))
18439 return SDValue();
18440 // Search for proper CA (non-zero) and CB that both are simm12.
18441 if ((C1 / C0) != 0 && isInt<12>(x: C1 / C0) && isInt<12>(x: C1 % C0) &&
18442 !isInt<12>(x: C0 * (C1 / C0))) {
18443 CA = C1 / C0;
18444 CB = C1 % C0;
18445 } else if ((C1 / C0 + 1) != 0 && isInt<12>(x: C1 / C0 + 1) &&
18446 isInt<12>(x: C1 % C0 - C0) && !isInt<12>(x: C0 * (C1 / C0 + 1))) {
18447 CA = C1 / C0 + 1;
18448 CB = C1 % C0 - C0;
18449 } else if ((C1 / C0 - 1) != 0 && isInt<12>(x: C1 / C0 - 1) &&
18450 isInt<12>(x: C1 % C0 + C0) && !isInt<12>(x: C0 * (C1 / C0 - 1))) {
18451 CA = C1 / C0 - 1;
18452 CB = C1 % C0 + C0;
18453 } else
18454 return SDValue();
18455 // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
18456 SDLoc DL(N);
18457 SDValue New0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0->getOperand(Num: 0),
18458 N2: DAG.getSignedConstant(Val: CA, DL, VT));
18459 SDValue New1 =
18460 DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: New0, N2: DAG.getSignedConstant(Val: C0, DL, VT));
18461 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: New1, N2: DAG.getSignedConstant(Val: CB, DL, VT));
18462}
18463
18464// add (zext, zext) -> zext (add (zext, zext))
18465// sub (zext, zext) -> sext (sub (zext, zext))
18466// mul (zext, zext) -> zext (mul (zext, zext))
18467// sdiv (zext, zext) -> zext (sdiv (zext, zext))
18468// udiv (zext, zext) -> zext (udiv (zext, zext))
18469// srem (zext, zext) -> zext (srem (zext, zext))
18470// urem (zext, zext) -> zext (urem (zext, zext))
18471//
18472// where the sum of the extend widths match, and the the range of the bin op
18473// fits inside the width of the narrower bin op. (For profitability on rvv, we
18474// use a power of two for both inner and outer extend.)
18475static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
18476
18477 EVT VT = N->getValueType(ResNo: 0);
18478 if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
18479 return SDValue();
18480
18481 SDValue N0 = N->getOperand(Num: 0);
18482 SDValue N1 = N->getOperand(Num: 1);
18483 if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
18484 return SDValue();
18485 if (!N0.hasOneUse() || !N1.hasOneUse())
18486 return SDValue();
18487
18488 SDValue Src0 = N0.getOperand(i: 0);
18489 SDValue Src1 = N1.getOperand(i: 0);
18490 EVT SrcVT = Src0.getValueType();
18491 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: SrcVT) ||
18492 SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 ||
18493 SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
18494 return SDValue();
18495
18496 LLVMContext &C = *DAG.getContext();
18497 EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(Context&: C);
18498 EVT NarrowVT = EVT::getVectorVT(Context&: C, VT: ElemVT, EC: VT.getVectorElementCount());
18499
18500 Src0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Src0), VT: NarrowVT, Operand: Src0);
18501 Src1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Src1), VT: NarrowVT, Operand: Src1);
18502
18503 // Src0 and Src1 are zero extended, so they're always positive if signed.
18504 //
18505 // sub can produce a negative from two positive operands, so it needs sign
18506 // extended. Other nodes produce a positive from two positive operands, so
18507 // zero extend instead.
18508 unsigned OuterExtend =
18509 N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
18510
18511 return DAG.getNode(
18512 Opcode: OuterExtend, DL: SDLoc(N), VT,
18513 Operand: DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NarrowVT, N1: Src0, N2: Src1));
18514}
18515
18516// Try to turn (add (xor bool, 1) -1) into (neg bool).
18517static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
18518 SDValue N0 = N->getOperand(Num: 0);
18519 SDValue N1 = N->getOperand(Num: 1);
18520 EVT VT = N->getValueType(ResNo: 0);
18521 SDLoc DL(N);
18522
18523 // RHS should be -1.
18524 if (!isAllOnesConstant(V: N1))
18525 return SDValue();
18526
18527 // Look for (xor X, 1).
18528 if (N0.getOpcode() != ISD::XOR || !isOneConstant(V: N0.getOperand(i: 1)))
18529 return SDValue();
18530
18531 // First xor input should be 0 or 1.
18532 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
18533 if (!DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask))
18534 return SDValue();
18535
18536 // Emit a negate of the setcc.
18537 return DAG.getNegative(Val: N0.getOperand(i: 0), DL, VT);
18538}
18539
18540// Fold (add X, (mulhs X, C)) -> (mulhsu X, C) if C is negative. This occurs
18541// in the expansion of sdiv i32 X, 7 using magic multiply.
18542//
18543// mulhs returns the hi from X * C = hi * 2^32 + lo.
18544//
18545// Since C<0, u(C) as an unsigned constant is 2^32 + C = u(C).
18546// mulhsu computes
18547// X * u(C0) = X * (C + 2^32)
18548// = X * 2^32 + C * X // C * X is the same as mulhs
18549// = X * 2^32 + hi * 2^32 + lo
18550// = (X + hi) * 2^32 + lo
18551// So mulhsu computes (X + hi).
18552static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
18553 const RISCVSubtarget &Subtarget) {
18554 EVT VT = N->getValueType(ResNo: 0);
18555 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18556 bool IsPExtPackedDoubleType =
18557 VT.isSimple() && Subtarget.isPExtPackedDoubleType(VT: VT.getSimpleVT());
18558 if (!TLI.isOperationLegal(Op: ISD::MULHS, VT) && !IsPExtPackedDoubleType &&
18559 !(Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8) &&
18560 !(Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::v8i8))
18561 return SDValue();
18562
18563 using namespace SDPatternMatch;
18564 SDValue X, Mulh;
18565 APInt C;
18566 if (!sd_match(N,
18567 P: m_Add(L: m_Value(N&: X),
18568 R: m_OneUse(P: m_Value(N&: Mulh, P: m_BinOp(Opc: ISD::MULHS, L: m_Deferred(V&: X),
18569 R: m_ConstInt(V&: C)))))) ||
18570 !C.isNegative())
18571 return SDValue();
18572
18573 SDLoc DL(N);
18574
18575 // We don't have a v4i8 MULHSU instruction, use a WMULSU+SRL+TRUNC.
18576 auto MakePWMulSU = [&](SDValue A, SDValue B) -> SDValue {
18577 SDValue WMul = DAG.getNode(Opcode: RISCVISD::PWMULSU, DL, VT: MVT::v4i16, N1: A, N2: B);
18578 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::v4i16, N1: WMul,
18579 N2: DAG.getConstant(Val: 8, DL, VT: MVT::v4i16));
18580 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::v4i8, Operand: Shifted);
18581 };
18582
18583 // We need to split double wide vectors ourselves, op legalization won't
18584 // run for custom nodes.
18585 if (IsPExtPackedDoubleType) {
18586 MVT HalfVT = VT.getSimpleVT().getHalfNumVectorElementsVT();
18587 auto [XLo, XHi] = DAG.SplitVector(N: X, DL, LoVT: HalfVT, HiVT: HalfVT);
18588 auto [CLo, CHi] = DAG.SplitVector(N: Mulh.getOperand(i: 1), DL, LoVT: HalfVT, HiVT: HalfVT);
18589 SDValue ResLo, ResHi;
18590 if (HalfVT == MVT::v4i8) {
18591 ResLo = MakePWMulSU(XLo, CLo);
18592 ResHi = MakePWMulSU(XHi, CHi);
18593 } else {
18594 ResLo = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: HalfVT, N1: XLo, N2: CLo);
18595 ResHi = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: HalfVT, N1: XHi, N2: CHi);
18596 }
18597 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: ResLo, N2: ResHi);
18598 }
18599
18600 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8)
18601 return MakePWMulSU(X, Mulh.getOperand(i: 1));
18602
18603 // We don't have a v8i8 MULHSU instruction on RV64 either; build it from a
18604 // pair of widening byte multiplies recombined with PPAIRO.
18605 if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::v8i8) {
18606 SDValue C = Mulh.getOperand(i: 1);
18607 SDValue Lo = DAG.getNode(Opcode: RISCVISD::PMULSU_HALVES_00, DL, VT: MVT::v4i16, N1: X, N2: C);
18608 SDValue Hi = DAG.getNode(Opcode: RISCVISD::PMULSU_HALVES_11, DL, VT: MVT::v4i16, N1: X, N2: C);
18609 return DAG.getNode(Opcode: RISCVISD::PPAIRO, DL, VT, N1: DAG.getBitcast(VT, V: Lo),
18610 N2: DAG.getBitcast(VT, V: Hi));
18611 }
18612
18613 return DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT, N1: X, N2: Mulh.getOperand(i: 1));
18614}
18615
18616// Fold an add of a multiply-parts product into the accumulating form.
18617static SDValue combineAddMulParts(SDNode *N, SelectionDAG &DAG,
18618 const RISCVSubtarget &Subtarget) {
18619 if (!Subtarget.hasStdExtP())
18620 return SDValue();
18621
18622 for (unsigned I = 0; I != 2; ++I) {
18623 SDValue Mul = N->getOperand(Num: I);
18624 if (Mul.getOpcode() != ISD::INTRINSIC_WO_CHAIN || !Mul.hasOneUse())
18625 continue;
18626 Intrinsic::ID AccId =
18627 getRVPMulPartsAccIntrinsic(IntNo: Mul.getConstantOperandVal(i: 0));
18628 if (AccId == Intrinsic::not_intrinsic)
18629 continue;
18630
18631 SDLoc DL(N);
18632 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: N->getValueType(ResNo: 0),
18633 N1: DAG.getTargetConstant(Val: AccId, DL, VT: Subtarget.getXLenVT()),
18634 N2: N->getOperand(Num: 1 - I), N3: Mul.getOperand(i: 1),
18635 N4: Mul.getOperand(i: 2));
18636 }
18637 return SDValue();
18638}
18639
18640static SDValue combinePExtWideningAddSub(SDNode *N, SelectionDAG &DAG,
18641 const RISCVSubtarget &Subtarget) {
18642 // Recognize the RV64 decompositions listed for the 32-bit packed widening
18643 // add/sub operations. Mixed signed/unsigned PM2 forms are outside that group.
18644 if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit())
18645 return SDValue();
18646
18647 unsigned Opcode = N->getOpcode();
18648 if (Opcode != ISD::ADD && Opcode != ISD::SUB)
18649 return SDValue();
18650
18651 EVT VT = N->getValueType(ResNo: 0);
18652 if (VT != MVT::v4i16 && VT != MVT::v2i32)
18653 return SDValue();
18654
18655 SDValue N0 = N->getOperand(Num: 0);
18656 SDValue N1 = N->getOperand(Num: 1);
18657 unsigned ExtendOpcode = N0.getOpcode();
18658 if ((ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND) ||
18659 N1.getOpcode() != ExtendOpcode || !N0.hasOneUse() || !N1.hasOneUse())
18660 return SDValue();
18661 bool IsSExt = ExtendOpcode == ISD::SIGN_EXTEND;
18662
18663 SDValue A = N0.getOperand(i: 0);
18664 SDValue B = N1.getOperand(i: 0);
18665 MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
18666 MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
18667 if (A.getValueType() != SrcVT || B.getValueType() != SrcVT)
18668 return SDValue();
18669
18670 if (VT == MVT::v4i16 && !IsSExt)
18671 return SDValue();
18672
18673 bool IsPM2Halfword = VT == MVT::v2i32 && (IsSExt || Opcode == ISD::ADD);
18674 if (VT == MVT::v2i32 && !IsPM2Halfword)
18675 return SDValue();
18676
18677 SDLoc DL(N);
18678 A = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: LegalSrcVT, N1: A, N2: DAG.getUNDEF(VT: SrcVT));
18679 B = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: LegalSrcVT, N1: B, N2: DAG.getUNDEF(VT: SrcVT));
18680
18681 SDValue Zip = DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT: LegalSrcVT, N1: A, N2: B);
18682 if (VT == MVT::v4i16) {
18683 SDValue ZipAsVT = DAG.getBitcast(VT, V: Zip);
18684 SDValue Low = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: ZipAsVT,
18685 N2: DAG.getValueType(MVT::v4i8));
18686 SDValue High = DAG.getNode(Opcode: RISCVISD::PSRA, DL, VT, N1: ZipAsVT,
18687 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i64));
18688 return DAG.getNode(Opcode, DL, VT, N1: Low, N2: High);
18689 }
18690
18691 SDValue Ones = DAG.getConstant(Val: 1, DL, VT: LegalSrcVT);
18692 if (IsSExt) {
18693 unsigned Opc = Opcode == ISD::ADD ? RISCVISD::PM2ADD_H : RISCVISD::PM2SUB_H;
18694 return DAG.getNode(Opcode: Opc, DL, VT, N1: Zip, N2: Ones);
18695 }
18696
18697 if (Opcode == ISD::ADD)
18698 return DAG.getNode(Opcode: RISCVISD::PM2ADDU_H, DL, VT, N1: Zip, N2: Ones);
18699
18700 return SDValue();
18701}
18702
18703static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG,
18704 const RISCVSubtarget &Subtarget) {
18705 using namespace SDPatternMatch;
18706
18707 if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit())
18708 return SDValue();
18709
18710 if (N->getOpcode() != ISD::ADD)
18711 return SDValue();
18712
18713 MVT VT = N->getSimpleValueType(ResNo: 0);
18714 if (VT != MVT::v4i16 && VT != MVT::v2i32)
18715 return SDValue();
18716
18717 auto MatchExtend = [](SDValue V, unsigned ExtendOpcode, MVT SrcVT,
18718 SDValue &Src) {
18719 return sd_match(
18720 N: V, P: m_OneUse(P: m_Node(Opcode: ExtendOpcode, preds: m_Value(N&: Src, P: m_SpecificVT(RefVT: SrcVT)))));
18721 };
18722
18723 auto Match = [&](SDValue Ext, SDValue Add, SDValue &Acc, SDValue &A,
18724 SDValue &B, bool &IsSExt) {
18725 MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
18726 unsigned ExtendOpcode = Ext.getOpcode();
18727 if (ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND)
18728 return false;
18729
18730 if (!MatchExtend(Ext, ExtendOpcode, SrcVT, B))
18731 return false;
18732
18733 if (!sd_match(N: Add, P: m_OneUse(P: m_Add(L: m_Value(), R: m_Value()))))
18734 return false;
18735
18736 if (MatchExtend(Add.getOperand(i: 0), ExtendOpcode, SrcVT, A))
18737 Acc = Add.getOperand(i: 1);
18738 else if (MatchExtend(Add.getOperand(i: 1), ExtendOpcode, SrcVT, A))
18739 Acc = Add.getOperand(i: 0);
18740 else
18741 return false;
18742
18743 if (Acc.getValueType() != VT)
18744 return false;
18745
18746 IsSExt = ExtendOpcode == ISD::SIGN_EXTEND;
18747 return true;
18748 };
18749
18750 SDValue Acc, A, B;
18751 bool IsSExt;
18752 if (!Match(N->getOperand(Num: 0), N->getOperand(Num: 1), Acc, A, B, IsSExt) &&
18753 !Match(N->getOperand(Num: 1), N->getOperand(Num: 0), Acc, A, B, IsSExt))
18754 return SDValue();
18755
18756 if (VT == MVT::v4i16 && !IsSExt)
18757 return SDValue();
18758
18759 SDLoc DL(N);
18760 MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
18761 MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
18762 A = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: LegalSrcVT, N1: A, N2: DAG.getUNDEF(VT: SrcVT));
18763 B = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: LegalSrcVT, N1: B, N2: DAG.getUNDEF(VT: SrcVT));
18764
18765 SDValue Zip = DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT: LegalSrcVT, N1: A, N2: B);
18766 if (VT == MVT::v4i16) {
18767 SDValue ZipAsVT = DAG.getBitcast(VT, V: Zip);
18768 SDValue Low = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: ZipAsVT,
18769 N2: DAG.getValueType(MVT::v4i8));
18770 SDValue High = DAG.getNode(Opcode: RISCVISD::PSRA, DL, VT, N1: ZipAsVT,
18771 N2: DAG.getConstant(Val: 8, DL, VT: MVT::i64));
18772 return DAG.getNode(Opcode: ISD::ADD, DL, VT,
18773 N1: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Acc, N2: Low), N2: High);
18774 }
18775
18776 SDValue Ones = DAG.getConstant(Val: 1, DL, VT: LegalSrcVT);
18777 unsigned Opc = IsSExt ? RISCVISD::PM2ADDA_H : RISCVISD::PM2ADDAU_H;
18778 return DAG.getNode(Opcode: Opc, DL, VT, N1: Acc, N2: Zip, N3: Ones);
18779}
18780
18781static SDValue performADDCombine(SDNode *N,
18782 TargetLowering::DAGCombinerInfo &DCI,
18783 const RISCVSubtarget &Subtarget) {
18784 SelectionDAG &DAG = DCI.DAG;
18785 if (SDValue V = combineAddOfBooleanXor(N, DAG))
18786 return V;
18787 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
18788 return V;
18789 if (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) {
18790 if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
18791 return V;
18792 if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
18793 return V;
18794 }
18795 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
18796 return V;
18797 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
18798 return V;
18799 if (SDValue V = combinePExtWideningAddAcc(N, DAG, Subtarget))
18800 return V;
18801 if (SDValue V = combinePExtWideningAddSub(N, DAG, Subtarget))
18802 return V;
18803 if (SDValue V = combineBinOpOfZExt(N, DAG))
18804 return V;
18805 if (SDValue V = combineAddMulParts(N, DAG, Subtarget))
18806 return V;
18807 if (SDValue V = combineAddMulh(N, DAG, Subtarget))
18808 return V;
18809
18810 // fold (add (select lhs, rhs, cc, 0, y), x) ->
18811 // (select lhs, rhs, cc, x, (add x, y))
18812 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
18813}
18814
18815// Try to turn a sub boolean RHS and constant LHS into an addi.
18816static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
18817 SDValue N0 = N->getOperand(Num: 0);
18818 SDValue N1 = N->getOperand(Num: 1);
18819 EVT VT = N->getValueType(ResNo: 0);
18820 SDLoc DL(N);
18821
18822 // Require a constant LHS.
18823 auto *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
18824 if (!N0C)
18825 return SDValue();
18826
18827 // All our optimizations involve subtracting 1 from the immediate and forming
18828 // an ADDI. Make sure the new immediate is valid for an ADDI.
18829 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
18830 if (!ImmValMinus1.isSignedIntN(N: 12))
18831 return SDValue();
18832
18833 SDValue NewLHS;
18834 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
18835 // (sub constant, (setcc x, y, eq/neq)) ->
18836 // (add (setcc x, y, neq/eq), constant - 1)
18837 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
18838 EVT SetCCOpVT = N1.getOperand(i: 0).getValueType();
18839 if (!isIntEqualitySetCC(Code: CCVal) || !SetCCOpVT.isInteger())
18840 return SDValue();
18841 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
18842 NewLHS =
18843 DAG.getSetCC(DL: SDLoc(N1), VT, LHS: N1.getOperand(i: 0), RHS: N1.getOperand(i: 1), Cond: CCVal);
18844 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(V: N1.getOperand(i: 1)) &&
18845 N1.getOperand(i: 0).getOpcode() == ISD::SETCC) {
18846 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
18847 // Since setcc returns a bool the xor is equivalent to 1-setcc.
18848 NewLHS = N1.getOperand(i: 0);
18849 } else
18850 return SDValue();
18851
18852 SDValue NewRHS = DAG.getConstant(Val: ImmValMinus1, DL, VT);
18853 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewLHS, N2: NewRHS);
18854}
18855
18856// Looks for (sub (shl X, 8-Y), (shr X, Y)) where the Y-th bit in each byte is
18857// potentially set. It is fine for Y to be 0, meaning that (sub (shl X, 8), X)
18858// is also valid. Replace with (orc.b X). For example, 0b0000_1000_0000_1000 is
18859// valid with Y=3, while 0b0000_1000_0000_0100 is not.
18860static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
18861 const RISCVSubtarget &Subtarget) {
18862 if (!Subtarget.hasStdExtZbb())
18863 return SDValue();
18864
18865 EVT VT = N->getValueType(ResNo: 0);
18866
18867 if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
18868 return SDValue();
18869
18870 SDValue N0 = N->getOperand(Num: 0);
18871 SDValue N1 = N->getOperand(Num: 1);
18872
18873 if (N0->getOpcode() != ISD::SHL)
18874 return SDValue();
18875
18876 auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
18877 if (!ShAmtCLeft)
18878 return SDValue();
18879 unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
18880
18881 if (ShiftedAmount >= 8)
18882 return SDValue();
18883
18884 SDValue LeftShiftOperand = N0->getOperand(Num: 0);
18885 SDValue RightShiftOperand = N1;
18886
18887 if (ShiftedAmount != 0) { // Right operand must be a right shift.
18888 if (N1->getOpcode() != ISD::SRL)
18889 return SDValue();
18890 auto *ShAmtCRight = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
18891 if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
18892 return SDValue();
18893 RightShiftOperand = N1.getOperand(i: 0);
18894 }
18895
18896 // At least one shift should have a single use.
18897 if (!N0.hasOneUse() && (ShiftedAmount == 0 || !N1.hasOneUse()))
18898 return SDValue();
18899
18900 if (LeftShiftOperand != RightShiftOperand)
18901 return SDValue();
18902
18903 APInt Mask = APInt::getSplat(NewLen: VT.getSizeInBits(), V: APInt(8, 0x1));
18904 Mask <<= ShiftedAmount;
18905 // Check that X has indeed the right shape (only the Y-th bit can be set in
18906 // every byte).
18907 if (!DAG.MaskedValueIsZero(Op: LeftShiftOperand, Mask: ~Mask))
18908 return SDValue();
18909
18910 return DAG.getNode(Opcode: RISCVISD::ORC_B, DL: SDLoc(N), VT, Operand: LeftShiftOperand);
18911}
18912
18913static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
18914 const RISCVSubtarget &Subtarget) {
18915 if (SDValue V = combineSubOfBoolean(N, DAG))
18916 return V;
18917
18918 EVT VT = N->getValueType(ResNo: 0);
18919 SDValue N0 = N->getOperand(Num: 0);
18920 SDValue N1 = N->getOperand(Num: 1);
18921 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
18922 if (isNullConstant(V: N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
18923 isNullConstant(V: N1.getOperand(i: 1)) &&
18924 N1.getValueType() == N1.getOperand(i: 0).getValueType()) {
18925 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
18926 if (CCVal == ISD::SETLT) {
18927 SDLoc DL(N);
18928 unsigned ShAmt = N0.getValueSizeInBits() - 1;
18929 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N1.getOperand(i: 0),
18930 N2: DAG.getConstant(Val: ShAmt, DL, VT));
18931 }
18932 }
18933
18934 if (SDValue V = combinePExtWideningAddSub(N, DAG, Subtarget))
18935 return V;
18936 if (SDValue V = combineBinOpOfZExt(N, DAG))
18937 return V;
18938 if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget))
18939 return V;
18940
18941 // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
18942 // (select lhs, rhs, cc, x, (sub x, y))
18943 return combineSelectAndUse(N, Slct: N1, OtherOp: N0, DAG, /*AllOnes*/ false, Subtarget);
18944}
18945
18946// Apply DeMorgan's law to (and/or (xor X, 1), (xor Y, 1)) if X and Y are 0/1.
18947// Legalizing setcc can introduce xors like this. Doing this transform reduces
18948// the number of xors and may allow the xor to fold into a branch condition.
18949static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
18950 SDValue N0 = N->getOperand(Num: 0);
18951 SDValue N1 = N->getOperand(Num: 1);
18952 bool IsAnd = N->getOpcode() == ISD::AND;
18953
18954 if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR)
18955 return SDValue();
18956
18957 if (!N0.hasOneUse() || !N1.hasOneUse())
18958 return SDValue();
18959
18960 SDValue N01 = N0.getOperand(i: 1);
18961 SDValue N11 = N1.getOperand(i: 1);
18962
18963 // For AND, SimplifyDemandedBits may have turned one of the (xor X, 1) into
18964 // (xor X, -1) based on the upper bits of the other operand being 0. If the
18965 // operation is And, allow one of the Xors to use -1.
18966 if (isOneConstant(V: N01)) {
18967 if (!isOneConstant(V: N11) && !(IsAnd && isAllOnesConstant(V: N11)))
18968 return SDValue();
18969 } else if (isOneConstant(V: N11)) {
18970 // N01 and N11 being 1 was already handled. Handle N11==1 and N01==-1.
18971 if (!(IsAnd && isAllOnesConstant(V: N01)))
18972 return SDValue();
18973 } else
18974 return SDValue();
18975
18976 EVT VT = N->getValueType(ResNo: 0);
18977
18978 SDValue N00 = N0.getOperand(i: 0);
18979 SDValue N10 = N1.getOperand(i: 0);
18980
18981 // The LHS of the xors needs to be 0/1.
18982 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
18983 if (!DAG.MaskedValueIsZero(Op: N00, Mask) || !DAG.MaskedValueIsZero(Op: N10, Mask))
18984 return SDValue();
18985
18986 // Invert the opcode and insert a new xor.
18987 SDLoc DL(N);
18988 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
18989 SDValue Logic = DAG.getNode(Opcode: Opc, DL, VT, N1: N00, N2: N10);
18990 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Logic, N2: DAG.getConstant(Val: 1, DL, VT));
18991}
18992
18993// Fold (vXi8 (trunc (vselect (setltu, X, 256), X, (sext (setgt X, 0))))) to
18994// (vXi8 (trunc (smin (smax X, 0), 255))). This represents saturating a signed
18995// value to an unsigned value. This will be lowered to vmax and series of
18996// vnclipu instructions later. This can be extended to other truncated types
18997// other than i8 by replacing 256 and 255 with the equivalent constants for the
18998// type.
18999static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG) {
19000 EVT VT = N->getValueType(ResNo: 0);
19001 SDValue N0 = N->getOperand(Num: 0);
19002 EVT SrcVT = N0.getValueType();
19003
19004 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19005 if (!VT.isVector() || !TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
19006 return SDValue();
19007
19008 if (N0.getOpcode() != ISD::VSELECT || !N0.hasOneUse())
19009 return SDValue();
19010
19011 SDValue Cond = N0.getOperand(i: 0);
19012 SDValue True = N0.getOperand(i: 1);
19013 SDValue False = N0.getOperand(i: 2);
19014
19015 if (Cond.getOpcode() != ISD::SETCC)
19016 return SDValue();
19017
19018 // FIXME: Support the version of this pattern with the select operands
19019 // swapped.
19020 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
19021 if (CCVal != ISD::SETULT)
19022 return SDValue();
19023
19024 SDValue CondLHS = Cond.getOperand(i: 0);
19025 SDValue CondRHS = Cond.getOperand(i: 1);
19026
19027 if (CondLHS != True)
19028 return SDValue();
19029
19030 unsigned ScalarBits = VT.getScalarSizeInBits();
19031
19032 // FIXME: Support other constants.
19033 ConstantSDNode *CondRHSC = isConstOrConstSplat(N: CondRHS);
19034 if (!CondRHSC || CondRHSC->getAPIntValue() != (1ULL << ScalarBits))
19035 return SDValue();
19036
19037 if (False.getOpcode() != ISD::SIGN_EXTEND)
19038 return SDValue();
19039
19040 False = False.getOperand(i: 0);
19041
19042 if (False.getOpcode() != ISD::SETCC || False.getOperand(i: 0) != True)
19043 return SDValue();
19044
19045 ConstantSDNode *FalseRHSC = isConstOrConstSplat(N: False.getOperand(i: 1));
19046 if (!FalseRHSC || !FalseRHSC->isZero())
19047 return SDValue();
19048
19049 ISD::CondCode CCVal2 = cast<CondCodeSDNode>(Val: False.getOperand(i: 2))->get();
19050 if (CCVal2 != ISD::SETGT)
19051 return SDValue();
19052
19053 // Emit the signed to unsigned saturation pattern.
19054 SDLoc DL(N);
19055 SDValue Max =
19056 DAG.getNode(Opcode: ISD::SMAX, DL, VT: SrcVT, N1: True, N2: DAG.getConstant(Val: 0, DL, VT: SrcVT));
19057 SDValue Min =
19058 DAG.getNode(Opcode: ISD::UMIN, DL, VT: SrcVT, N1: Max,
19059 N2: DAG.getConstant(Val: (1ULL << ScalarBits) - 1, DL, VT: SrcVT));
19060 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Min);
19061}
19062
19063// Handle P extension truncate patterns, both on packed vectors and on scalar
19064// i32 (the RV32-only asub/asubu and mulhr* instructions):
19065// ASUB/ASUBU: (trunc (srl (sub ([s|z]ext a), ([s|z]ext b)), 1))
19066// MULHSU: (trunc (srl (mul (sext a), (zext b)), EltBits))
19067// MULHR*: (trunc (srl (add (mul (sext a), (zext b)), round_const), EltBits))
19068static SDValue combinePExtTruncate(SDNode *N, SelectionDAG &DAG,
19069 const RISCVSubtarget &Subtarget) {
19070 SDValue N0 = N->getOperand(Num: 0);
19071 EVT VT = N->getValueType(ResNo: 0);
19072 if (N0.getOpcode() != ISD::SRL)
19073 return SDValue();
19074
19075 if (VT != MVT::v4i16 && VT != MVT::v2i16 && VT != MVT::v8i8 &&
19076 VT != MVT::v4i8 && VT != MVT::v2i32 && VT != MVT::i32)
19077 return SDValue();
19078
19079 ConstantSDNode *C = isConstOrConstSplat(N: N0.getOperand(i: 1));
19080 if (!C)
19081 return SDValue();
19082
19083 SDValue Op = N0.getOperand(i: 0);
19084 unsigned ShAmtVal = C->getZExtValue();
19085 unsigned EltBits = VT.getScalarSizeInBits();
19086
19087 // Check for rounding pattern: (add (mul ...), round_const)
19088 bool IsRounding = false;
19089 if (Op.getOpcode() == ISD::ADD && (EltBits == 16 || EltBits == 32)) {
19090 ConstantSDNode *RndC = isConstOrConstSplat(N: Op.getOperand(i: 1));
19091 if (RndC && RndC->getZExtValue() == (1ULL << (EltBits - 1)) &&
19092 Op.getOperand(i: 0).getOpcode() == ISD::MUL) {
19093 Op = Op.getOperand(i: 0);
19094 IsRounding = true;
19095 }
19096 }
19097
19098 // Ensure Op is a binary operation before accessing its operands.
19099 if (Op.getNumOperands() != 2)
19100 return SDValue();
19101
19102 SDValue LHS = Op.getOperand(i: 0);
19103 SDValue RHS = Op.getOperand(i: 1);
19104
19105 bool LHSIsSExt = LHS.getOpcode() == ISD::SIGN_EXTEND;
19106 bool LHSIsZExt = LHS.getOpcode() == ISD::ZERO_EXTEND;
19107 bool RHSIsSExt = RHS.getOpcode() == ISD::SIGN_EXTEND;
19108 bool RHSIsZExt = RHS.getOpcode() == ISD::ZERO_EXTEND;
19109
19110 if (!(LHSIsSExt || LHSIsZExt) || !(RHSIsSExt || RHSIsZExt))
19111 return SDValue();
19112
19113 SDValue A = LHS.getOperand(i: 0);
19114 SDValue B = RHS.getOperand(i: 0);
19115
19116 if (A.getValueType() != VT || B.getValueType() != VT)
19117 return SDValue();
19118
19119 unsigned Opc;
19120 switch (Op.getOpcode()) {
19121 default:
19122 return SDValue();
19123 case ISD::SUB:
19124 // PASUB/PASUBU: shift amount must be 1
19125 if (ShAmtVal != 1)
19126 return SDValue();
19127 if (LHSIsSExt && RHSIsSExt)
19128 Opc = RISCVISD::ASUB;
19129 else if (LHSIsZExt && RHSIsZExt)
19130 Opc = RISCVISD::ASUBU;
19131 else
19132 return SDValue();
19133 break;
19134 case ISD::MUL:
19135 // MULH*/MULHR*: shift amount must be element size, only for i16/i32
19136 if (ShAmtVal != EltBits || (EltBits != 16 && EltBits != 32))
19137 return SDValue();
19138 if (!Subtarget.is64Bit() && VT == MVT::v4i16)
19139 return SDValue();
19140 if (IsRounding) {
19141 if (LHSIsSExt && RHSIsSExt) {
19142 Opc = RISCVISD::MULHR;
19143 } else if (LHSIsZExt && RHSIsZExt) {
19144 Opc = RISCVISD::MULHRU;
19145 } else if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
19146 Opc = RISCVISD::MULHRSU;
19147 // commuted case
19148 if (LHSIsZExt && RHSIsSExt)
19149 std::swap(a&: A, b&: B);
19150 } else {
19151 return SDValue();
19152 }
19153 } else {
19154 // Scalar mulhsu is handled elsewhere, only match the packed MULHSU here.
19155 if (!VT.isVector())
19156 return SDValue();
19157 if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
19158 Opc = RISCVISD::MULHSU;
19159 // commuted case
19160 if (LHSIsZExt && RHSIsSExt)
19161 std::swap(a&: A, b&: B);
19162 } else
19163 return SDValue();
19164 }
19165
19166 // RV32 has scalar rounded multiply-high instructions, but no paired form.
19167 // Split v2i32 while the widening multiply shape is still visible.
19168 if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
19169 if (!IsRounding)
19170 return SDValue();
19171 SDLoc DL(N);
19172 SDValue ALo = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: A, Idx: 0);
19173 SDValue AHi = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: A, Idx: 1);
19174 SDValue BLo = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: B, Idx: 0);
19175 SDValue BHi = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: B, Idx: 1);
19176 SDValue Lo = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, N1: ALo, N2: BLo);
19177 SDValue Hi = DAG.getNode(Opcode: Opc, DL, VT: MVT::i32, N1: AHi, N2: BHi);
19178 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
19179 }
19180
19181 // On RV64, v8i8 MULHSU is built from a pair of widening byte multiplies
19182 // (picking out the even/odd result lanes) recombined with PPAIRO.
19183 if (Subtarget.is64Bit() && VT == MVT::v8i8 && Opc == RISCVISD::MULHSU) {
19184 SDLoc DL(N);
19185 SDValue Lo =
19186 DAG.getNode(Opcode: RISCVISD::PMULSU_HALVES_00, DL, VT: MVT::v4i16, N1: A, N2: B);
19187 SDValue Hi =
19188 DAG.getNode(Opcode: RISCVISD::PMULSU_HALVES_11, DL, VT: MVT::v4i16, N1: A, N2: B);
19189 return DAG.getNode(Opcode: RISCVISD::PPAIRO, DL, VT, N1: DAG.getBitcast(VT, V: Lo),
19190 N2: DAG.getBitcast(VT, V: Hi));
19191 }
19192 break;
19193 }
19194
19195 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT, Ops: {A, B});
19196}
19197
19198static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
19199 const RISCVSubtarget &Subtarget) {
19200 SDValue N0 = N->getOperand(Num: 0);
19201 EVT VT = N->getValueType(ResNo: 0);
19202
19203 // P truncate patterns: packed vectors, plus RV32-only scalar i32.
19204 if (Subtarget.hasStdExtP() &&
19205 (VT.isFixedLengthVector() || (VT == MVT::i32 && !Subtarget.is64Bit())))
19206 return combinePExtTruncate(N, DAG, Subtarget);
19207
19208 // Pre-promote (i1 (truncate (srl X, Y))) on RV64 with Zbs without zero
19209 // extending X. This is safe since we only need the LSB after the shift and
19210 // shift amounts larger than 31 would produce poison. If we wait until
19211 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
19212 // to use a BEXT instruction.
19213 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
19214 N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL &&
19215 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) && N0.hasOneUse()) {
19216 SDLoc DL(N0);
19217 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
19218 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
19219 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
19220 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT, Operand: Srl);
19221 }
19222
19223 return combineTruncSelectToSMaxUSat(N, DAG);
19224}
19225
19226// InstCombinerImpl::transformZExtICmp will narrow a zext of an icmp with a
19227// truncation. But RVV doesn't have truncation instructions for more than twice
19228// the bitwidth.
19229//
19230// E.g. trunc <vscale x 1 x i64> %x to <vscale x 1 x i8> will generate:
19231//
19232// vsetvli a0, zero, e32, m2, ta, ma
19233// vnsrl.wi v12, v8, 0
19234// vsetvli zero, zero, e16, m1, ta, ma
19235// vnsrl.wi v8, v12, 0
19236// vsetvli zero, zero, e8, mf2, ta, ma
19237// vnsrl.wi v8, v8, 0
19238//
19239// So reverse the combine so we generate an vmseq/vmsne again:
19240//
19241// and (lshr (trunc X), ShAmt), 1
19242// -->
19243// zext (icmp ne (and X, (1 << ShAmt)), 0)
19244//
19245// and (lshr (not (trunc X)), ShAmt), 1
19246// -->
19247// zext (icmp eq (and X, (1 << ShAmt)), 0)
19248static SDValue reverseZExtICmpCombine(SDNode *N, SelectionDAG &DAG,
19249 const RISCVSubtarget &Subtarget) {
19250 using namespace SDPatternMatch;
19251 SDLoc DL(N);
19252
19253 if (!Subtarget.hasVInstructions())
19254 return SDValue();
19255
19256 EVT VT = N->getValueType(ResNo: 0);
19257 if (!VT.isVector())
19258 return SDValue();
19259
19260 APInt ShAmt;
19261 SDValue Inner;
19262 if (!sd_match(N, P: m_And(L: m_OneUse(P: m_Srl(L: m_Value(N&: Inner), R: m_ConstInt(V&: ShAmt))),
19263 R: m_One())))
19264 return SDValue();
19265
19266 SDValue X;
19267 bool IsNot;
19268 if (sd_match(N: Inner, P: m_Not(V: m_Trunc(Op: m_Value(N&: X)))))
19269 IsNot = true;
19270 else if (sd_match(N: Inner, P: m_Trunc(Op: m_Value(N&: X))))
19271 IsNot = false;
19272 else
19273 return SDValue();
19274
19275 EVT WideVT = X.getValueType();
19276 if (VT.getScalarSizeInBits() >= WideVT.getScalarSizeInBits() / 2)
19277 return SDValue();
19278
19279 SDValue Res =
19280 DAG.getNode(Opcode: ISD::AND, DL, VT: WideVT, N1: X,
19281 N2: DAG.getConstant(Val: 1ULL << ShAmt.getZExtValue(), DL, VT: WideVT));
19282 Res = DAG.getSetCC(DL,
19283 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
19284 EC: WideVT.getVectorElementCount()),
19285 LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: WideVT),
19286 Cond: IsNot ? ISD::SETEQ : ISD::SETNE);
19287 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Res);
19288}
19289
19290static SDValue reduceANDOfAtomicLoad(SDNode *N,
19291 TargetLowering::DAGCombinerInfo &DCI) {
19292 SelectionDAG &DAG = DCI.DAG;
19293 if (N->getOpcode() != ISD::AND)
19294 return SDValue();
19295
19296 SDValue N0 = N->getOperand(Num: 0);
19297 if (N0.getOpcode() != ISD::ATOMIC_LOAD)
19298 return SDValue();
19299 if (!N0.hasOneUse())
19300 return SDValue();
19301
19302 AtomicSDNode *ALoad = cast<AtomicSDNode>(Val: N0.getNode());
19303 if (isStrongerThanMonotonic(AO: ALoad->getSuccessOrdering()))
19304 return SDValue();
19305
19306 EVT LoadedVT = ALoad->getMemoryVT();
19307 ConstantSDNode *MaskConst = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
19308 if (!MaskConst)
19309 return SDValue();
19310 uint64_t Mask = MaskConst->getZExtValue();
19311 uint64_t ExpectedMask = maskTrailingOnes<uint64_t>(N: LoadedVT.getSizeInBits());
19312 if (Mask != ExpectedMask)
19313 return SDValue();
19314
19315 SDValue ZextLoad = DAG.getAtomicLoad(
19316 ExtType: ISD::ZEXTLOAD, dl: SDLoc(N), MemVT: ALoad->getMemoryVT(), VT: N->getValueType(ResNo: 0),
19317 Chain: ALoad->getChain(), Ptr: ALoad->getBasePtr(), MMO: ALoad->getMemOperand());
19318 DCI.CombineTo(N, Res: ZextLoad);
19319 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N0.getNode(), 1), To: ZextLoad.getValue(R: 1));
19320 DCI.recursivelyDeleteUnusedNodes(N: N0.getNode());
19321 return SDValue(N, 0);
19322}
19323
19324// Sometimes a mask is applied after a shift. If that shift was fed by a
19325// load, there is sometimes the opportunity to narrow the load, which is
19326// hidden by the intermediate shift. Detect that case and commute the
19327// shift/and in order to enable load narrowing.
19328static SDValue combineNarrowableShiftedLoad(SDNode *N, SelectionDAG &DAG) {
19329 EVT VT = N->getValueType(ResNo: 0);
19330 if (!VT.isScalarInteger())
19331 return SDValue();
19332
19333 using namespace SDPatternMatch;
19334 SDValue LoadNode;
19335 APInt MaskVal, ShiftVal;
19336 // (and (shl (load ...), ShiftAmt), Mask)
19337 if (!sd_match(
19338 N, P: m_And(L: m_OneUse(P: m_Shl(L: m_Value(N&: LoadNode, P: m_SpecificOpc(Opcode: ISD::LOAD)),
19339 R: m_ConstInt(V&: ShiftVal))),
19340 R: m_ConstInt(V&: MaskVal)))) {
19341 return SDValue();
19342 }
19343
19344 uint64_t ShiftAmt = ShiftVal.getZExtValue();
19345
19346 if (ShiftAmt >= VT.getSizeInBits())
19347 return SDValue();
19348
19349 // Calculate the appropriate mask if it were applied before the shift.
19350 APInt InnerMask = MaskVal.lshr(shiftAmt: ShiftAmt);
19351 bool IsNarrowable =
19352 InnerMask == 0xff || InnerMask == 0xffff || InnerMask == 0xffffffff;
19353
19354 if (!IsNarrowable)
19355 return SDValue();
19356
19357 // AND the loaded value and change the shift appropriately, allowing
19358 // the load to be narrowed.
19359 SDLoc DL(N);
19360 SDValue InnerAnd = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoadNode,
19361 N2: DAG.getConstant(Val: InnerMask, DL, VT));
19362 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: InnerAnd,
19363 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
19364}
19365
19366// Combines two comparison operation and logic operation to one selection
19367// operation(min, max) and logic operation. Returns new constructed Node if
19368// conditions for optimization are satisfied.
19369static SDValue performANDCombine(SDNode *N,
19370 TargetLowering::DAGCombinerInfo &DCI,
19371 const RISCVSubtarget &Subtarget) {
19372 SelectionDAG &DAG = DCI.DAG;
19373 SDValue N0 = N->getOperand(Num: 0);
19374
19375 // Pre-promote (i32 (and (srl X, Y), 1)) on RV64 with Zbs without zero
19376 // extending X. This is safe since we only need the LSB after the shift and
19377 // shift amounts larger than 31 would produce poison. If we wait until
19378 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
19379 // to use a BEXT instruction.
19380 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
19381 N->getValueType(ResNo: 0) == MVT::i32 && isOneConstant(V: N->getOperand(Num: 1)) &&
19382 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
19383 N0.hasOneUse()) {
19384 SDLoc DL(N);
19385 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
19386 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
19387 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
19388 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Srl,
19389 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
19390 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: And);
19391 }
19392
19393 if (SDValue V = combineNarrowableShiftedLoad(N, DAG))
19394 return V;
19395 if (SDValue V = reverseZExtICmpCombine(N, DAG, Subtarget))
19396 return V;
19397 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
19398 return V;
19399 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
19400 return V;
19401 if (SDValue V = reduceANDOfAtomicLoad(N, DCI))
19402 return V;
19403
19404 if (DCI.isAfterLegalizeDAG())
19405 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
19406 return V;
19407
19408 // fold (and (select lhs, rhs, cc, -1, y), x) ->
19409 // (select lhs, rhs, cc, x, (and x, y))
19410 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget);
19411}
19412
19413// Try to pull an xor with 1 through a select idiom that uses czero_eqz/nez.
19414// FIXME: Generalize to other binary operators with same operand.
19415static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1,
19416 SelectionDAG &DAG) {
19417 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
19418
19419 if (N0.getOpcode() != RISCVISD::CZERO_EQZ ||
19420 N1.getOpcode() != RISCVISD::CZERO_NEZ ||
19421 !N0.hasOneUse() || !N1.hasOneUse())
19422 return SDValue();
19423
19424 // Should have the same condition.
19425 SDValue Cond = N0.getOperand(i: 1);
19426 if (Cond != N1.getOperand(i: 1))
19427 return SDValue();
19428
19429 SDValue TrueV = N0.getOperand(i: 0);
19430 SDValue FalseV = N1.getOperand(i: 0);
19431
19432 if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR ||
19433 TrueV.getOperand(i: 1) != FalseV.getOperand(i: 1) ||
19434 !isOneConstant(V: TrueV.getOperand(i: 1)) ||
19435 !TrueV.hasOneUse() || !FalseV.hasOneUse())
19436 return SDValue();
19437
19438 EVT VT = N->getValueType(ResNo: 0);
19439 SDLoc DL(N);
19440
19441 SDValue NewN0 = DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV.getOperand(i: 0),
19442 N2: Cond);
19443 SDValue NewN1 =
19444 DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV.getOperand(i: 0), N2: Cond);
19445 SDValue NewOr =
19446 DAG.getNode(Opcode: ISD::OR, DL, VT, N1: NewN0, N2: NewN1, Flags: SDNodeFlags::Disjoint);
19447 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewOr, N2: TrueV.getOperand(i: 1));
19448}
19449
19450// (xor X, (xor (and X, C2), Y))
19451// ->(qc_insb X, (sra Y, ShAmt), Width, ShAmt)
19452// where C2 is a shifted mask with width = Width and shift = ShAmt
19453// qc_insb might become qc.insb or qc.insbi depending on the operands.
19454static SDValue combineXorToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
19455 const RISCVSubtarget &Subtarget) {
19456 if (!Subtarget.hasVendorXqcibm())
19457 return SDValue();
19458
19459 using namespace SDPatternMatch;
19460 SDValue Base, Inserted;
19461 APInt CMask;
19462 if (!sd_match(N, P: m_Xor(L: m_Value(N&: Base),
19463 R: m_OneUse(P: m_Xor(L: m_OneUse(P: m_And(L: m_Deferred(V&: Base),
19464 R: m_ConstInt(V&: CMask))),
19465 R: m_Value(N&: Inserted))))))
19466 return SDValue();
19467
19468 if (N->getValueType(ResNo: 0) != MVT::i32)
19469 return SDValue();
19470 unsigned Width, ShAmt;
19471 if (!CMask.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width))
19472 return SDValue();
19473
19474 // Check if all zero bits in CMask are also zero in Inserted
19475 if (!DAG.MaskedValueIsZero(Op: Inserted, Mask: ~CMask))
19476 return SDValue();
19477
19478 SDLoc DL(N);
19479
19480 // `Inserted` needs to be right shifted before it is put into the
19481 // instruction.
19482 Inserted = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i32, N1: Inserted,
19483 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
19484
19485 SDValue Ops[] = {Base, Inserted, DAG.getConstant(Val: Width, DL, VT: MVT::i32),
19486 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
19487 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
19488}
19489
19490static SDValue combineOrToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
19491 const RISCVSubtarget &Subtarget) {
19492 if (!Subtarget.hasVendorXqcibm())
19493 return SDValue();
19494
19495 using namespace SDPatternMatch;
19496
19497 SDValue X;
19498 APInt MaskImm;
19499 if (!sd_match(N, P: m_Or(L: m_OneUse(P: m_Value(N&: X)), R: m_ConstInt(V&: MaskImm))))
19500 return SDValue();
19501
19502 unsigned ShAmt, Width;
19503 if (!MaskImm.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width) || MaskImm.isSignedIntN(N: 12))
19504 return SDValue();
19505
19506 if (N->getValueType(ResNo: 0) != MVT::i32)
19507 return SDValue();
19508
19509 // If Zbs is enabled and it is a single bit set we can use BSETI which
19510 // can be compressed to C_BSETI when Xqcibm in enabled.
19511 if (Width == 1 && Subtarget.hasStdExtZbs())
19512 return SDValue();
19513
19514 // If C1 is a shifted mask (but can't be formed as an ORI),
19515 // use a bitfield insert of -1.
19516 // Transform (or x, C1)
19517 // -> (qc.insbi x, -1, width, shift)
19518 SDLoc DL(N);
19519
19520 SDValue Ops[] = {X, DAG.getSignedConstant(Val: -1, DL, VT: MVT::i32),
19521 DAG.getConstant(Val: Width, DL, VT: MVT::i32),
19522 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
19523 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
19524}
19525
19526// Generate a QC_INSB/QC_INSBI from 'or (and X, MaskImm), OrImm' iff the value
19527// being inserted only sets known zero bits.
19528static SDValue combineOrAndToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
19529 const RISCVSubtarget &Subtarget) {
19530 // Supported only in Xqcibm for now.
19531 if (!Subtarget.hasVendorXqcibm())
19532 return SDValue();
19533
19534 using namespace SDPatternMatch;
19535
19536 SDValue Inserted;
19537 APInt MaskImm, OrImm;
19538 if (!sd_match(
19539 N, P: m_SpecificVT(RefVT: MVT::i32, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: Inserted),
19540 R: m_ConstInt(V&: MaskImm))),
19541 R: m_ConstInt(V&: OrImm)))))
19542 return SDValue();
19543
19544 // Compute the Known Zero for the AND as this allows us to catch more general
19545 // cases than just looking for AND with imm.
19546 KnownBits Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
19547
19548 // The bits being inserted must only set those bits that are known to be
19549 // zero.
19550 if (!OrImm.isSubsetOf(RHS: Known.Zero)) {
19551 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
19552 // currently handle this case.
19553 return SDValue();
19554 }
19555
19556 unsigned ShAmt, Width;
19557 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
19558 if (!Known.Zero.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width))
19559 return SDValue();
19560
19561 // QC_INSB(I) dst, src, #width, #shamt.
19562 SDLoc DL(N);
19563
19564 SDValue ImmNode =
19565 DAG.getSignedConstant(Val: OrImm.getSExtValue() >> ShAmt, DL, VT: MVT::i32);
19566
19567 SDValue Ops[] = {Inserted, ImmNode, DAG.getConstant(Val: Width, DL, VT: MVT::i32),
19568 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
19569 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
19570}
19571
19572static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
19573 const RISCVSubtarget &Subtarget) {
19574 SelectionDAG &DAG = DCI.DAG;
19575
19576 if (SDValue V = combineOrAndToBitfieldInsert(N, DAG, Subtarget))
19577 return V;
19578 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
19579 return V;
19580 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
19581 return V;
19582
19583 if (DCI.isAfterLegalizeDAG()) {
19584 if (SDValue V = combineOrToBitfieldInsert(N, DAG, Subtarget))
19585 return V;
19586 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
19587 return V;
19588 }
19589
19590 // Look for Or of CZERO_EQZ/NEZ with same condition which is the select idiom.
19591 // We may be able to pull a common operation out of the true and false value.
19592 SDValue N0 = N->getOperand(Num: 0);
19593 SDValue N1 = N->getOperand(Num: 1);
19594 if (SDValue V = combineOrOfCZERO(N, N0, N1, DAG))
19595 return V;
19596 if (SDValue V = combineOrOfCZERO(N, N0: N1, N1: N0, DAG))
19597 return V;
19598
19599 // fold (or (select cond, 0, y), x) ->
19600 // (select cond, x, (or x, y))
19601 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
19602}
19603
19604static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
19605 const RISCVSubtarget &Subtarget) {
19606 SDValue N0 = N->getOperand(Num: 0);
19607 SDValue N1 = N->getOperand(Num: 1);
19608
19609 // Pre-promote (i32 (xor (shl -1, X), ~0)) on RV64 with Zbs so we can use
19610 // (ADDI (BSET X0, X), -1). If we wait until type legalization, we'll create
19611 // RISCVISD:::SLLW and we can't recover it to use a BSET instruction.
19612 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
19613 N->getValueType(ResNo: 0) == MVT::i32 && isAllOnesConstant(V: N1) &&
19614 N0.getOpcode() == ISD::SHL && isAllOnesConstant(V: N0.getOperand(i: 0)) &&
19615 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) && N0.hasOneUse()) {
19616 SDLoc DL(N);
19617 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
19618 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
19619 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
19620 SDValue Not = DAG.getNOT(DL, Val: Shl, VT: MVT::i64);
19621 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Not);
19622 }
19623
19624 // fold (xor (sllw 1, x), -1) -> (rolw ~1, x)
19625 // NOTE: Assumes ROL being legal means ROLW is legal.
19626 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19627 if (N0.getOpcode() == RISCVISD::SLLW &&
19628 isAllOnesConstant(V: N1) && isOneConstant(V: N0.getOperand(i: 0)) &&
19629 TLI.isOperationLegal(Op: ISD::ROTL, VT: MVT::i64)) {
19630 SDLoc DL(N);
19631 return DAG.getNode(Opcode: RISCVISD::ROLW, DL, VT: MVT::i64,
19632 N1: DAG.getConstant(Val: ~1, DL, VT: MVT::i64), N2: N0.getOperand(i: 1));
19633 }
19634
19635 // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
19636 if (N0.getOpcode() == ISD::SETCC && isOneConstant(V: N1) && N0.hasOneUse()) {
19637 auto *ConstN00 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
19638 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
19639 if (ConstN00 && CC == ISD::SETLT) {
19640 EVT VT = N0.getValueType();
19641 SDLoc DL(N0);
19642 const APInt &Imm = ConstN00->getAPIntValue();
19643 if ((Imm + 1).isSignedIntN(N: 12))
19644 return DAG.getSetCC(DL, VT, LHS: N0.getOperand(i: 1),
19645 RHS: DAG.getConstant(Val: Imm + 1, DL, VT), Cond: CC);
19646 }
19647 }
19648
19649 if (SDValue V = combineXorToBitfieldInsert(N, DAG, Subtarget))
19650 return V;
19651
19652 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
19653 return V;
19654 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
19655 return V;
19656
19657 // fold (xor (select cond, 0, y), x) ->
19658 // (select cond, x, (xor x, y))
19659 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
19660}
19661
19662// Try to expand a multiply to a sequence of shifts and add/subs,
19663// for a machine without native mul instruction.
19664static SDValue expandMulToNAFSequence(SDNode *N, SelectionDAG &DAG,
19665 uint64_t MulAmt) {
19666 SDLoc DL(N);
19667 EVT VT = N->getValueType(ResNo: 0);
19668 const uint64_t BitWidth = VT.getFixedSizeInBits();
19669
19670 SDValue Result = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
19671 SDValue N0 = N->getOperand(Num: 0);
19672
19673 // Find the Non-adjacent form of the multiplier.
19674 for (uint64_t E = MulAmt, I = 0; E && I < BitWidth; ++I, E >>= 1) {
19675 if (E & 1) {
19676 bool IsAdd = (E & 3) == 1;
19677 E -= IsAdd ? 1 : -1;
19678 SDValue ShiftVal = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
19679 N2: DAG.getShiftAmountConstant(Val: I, VT, DL));
19680 ISD::NodeType AddSubOp = IsAdd ? ISD::ADD : ISD::SUB;
19681 Result = DAG.getNode(Opcode: AddSubOp, DL, VT, N1: Result, N2: ShiftVal);
19682 }
19683 }
19684
19685 return Result;
19686}
19687
19688// X * (2^N +/- 2^M) -> (add/sub (shl X, C1), (shl X, C2))
19689static SDValue expandMulToAddOrSubOfShl(SDNode *N, SelectionDAG &DAG,
19690 uint64_t MulAmt) {
19691 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
19692 SDValue X = N->getOperand(Num: 0);
19693 ISD::NodeType Op;
19694 uint64_t ShiftAmt1;
19695 bool CanSub = isPowerOf2_64(Value: MulAmt + MulAmtLowBit);
19696 auto PreferSub = [X, MulAmtLowBit]() {
19697 // For MulAmt == 3 << M both (X << M + 2) - (X << M)
19698 // and (X << M + 1) + (X << M) are valid expansions.
19699 // Prefer SUB if we can get (X << M + 2) for free,
19700 // because X is exact (Y >> M + 2).
19701 uint64_t ShAmt = Log2_64(Value: MulAmtLowBit) + 2;
19702 using namespace SDPatternMatch;
19703 return sd_match(N: X, P: m_ExactSr(L: m_Value(), R: m_SpecificInt(V: ShAmt)));
19704 };
19705 if (isPowerOf2_64(Value: MulAmt - MulAmtLowBit) && !(CanSub && PreferSub())) {
19706 Op = ISD::ADD;
19707 ShiftAmt1 = MulAmt - MulAmtLowBit;
19708 } else if (CanSub) {
19709 Op = ISD::SUB;
19710 ShiftAmt1 = MulAmt + MulAmtLowBit;
19711 } else {
19712 return SDValue();
19713 }
19714 EVT VT = N->getValueType(ResNo: 0);
19715 SDLoc DL(N);
19716 SDValue Shift1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X,
19717 N2: DAG.getConstant(Val: Log2_64(Value: ShiftAmt1), DL, VT));
19718 SDValue Shift2 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X,
19719 N2: DAG.getConstant(Val: Log2_64(Value: MulAmtLowBit), DL, VT));
19720 return DAG.getNode(Opcode: Op, DL, VT, N1: Shift1, N2: Shift2);
19721}
19722
19723static SDValue getShlAddShlAdd(SDNode *N, SelectionDAG &DAG, unsigned ShX,
19724 unsigned ShY, bool AddX, unsigned Shift) {
19725 SDLoc DL(N);
19726 EVT VT = N->getValueType(ResNo: 0);
19727 SDValue X = N->getOperand(Num: 0);
19728 // Put the shift first if we can fold:
19729 // a. a zext into the shift forming a slli.uw
19730 // b. an exact shift right forming one shorter shift or no shift at all
19731 using namespace SDPatternMatch;
19732 if (Shift != 0 &&
19733 sd_match(N: X, P: m_AnyOf(preds: m_And(L: m_Value(), R: m_SpecificInt(UINT64_C(0xffffffff))),
19734 preds: m_ExactSr(L: m_Value(), R: m_ConstInt())))) {
19735 X = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: Shift, DL, VT));
19736 Shift = 0;
19737 }
19738 SDValue ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
19739 N2: DAG.getTargetConstant(Val: ShY, DL, VT), N3: X);
19740 if (ShX != 0)
19741 ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: ShlAdd,
19742 N2: DAG.getTargetConstant(Val: ShX, DL, VT), N3: AddX ? X : ShlAdd);
19743 if (Shift == 0)
19744 return ShlAdd;
19745 // Otherwise, put the shl last so that it can fold with following instructions
19746 // (e.g. sext or add).
19747 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShlAdd, N2: DAG.getConstant(Val: Shift, DL, VT));
19748}
19749
19750static SDValue expandMulToShlAddShlAdd(SDNode *N, SelectionDAG &DAG,
19751 uint64_t MulAmt, unsigned Shift) {
19752 switch (MulAmt) {
19753 // 3/5/9 -> (shYadd X, X)
19754 case 3:
19755 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 1, /*AddX=*/false, Shift);
19756 case 5:
19757 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 2, /*AddX=*/false, Shift);
19758 case 9:
19759 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 3, /*AddX=*/false, Shift);
19760 // 3/5/9 * 3/5/9 -> (shXadd (shYadd X, X), (shYadd X, X))
19761 case 5 * 3:
19762 return getShlAddShlAdd(N, DAG, ShX: 2, ShY: 1, /*AddX=*/false, Shift);
19763 case 9 * 3:
19764 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 1, /*AddX=*/false, Shift);
19765 case 5 * 5:
19766 return getShlAddShlAdd(N, DAG, ShX: 2, ShY: 2, /*AddX=*/false, Shift);
19767 case 9 * 5:
19768 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 2, /*AddX=*/false, Shift);
19769 case 9 * 9:
19770 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 3, /*AddX=*/false, Shift);
19771 default:
19772 break;
19773 }
19774
19775 int ShX;
19776 if (int ShY = isShifted359(Value: MulAmt - 1, Shift&: ShX)) {
19777 assert(ShX != 0 && "MulAmt=4,6,10 handled before");
19778 // 2/4/8 * 3/5/9 + 1 -> (shXadd (shYadd X, X), X)
19779 if (ShX <= 3)
19780 return getShlAddShlAdd(N, DAG, ShX, ShY, /*AddX=*/true, Shift);
19781 // 2^N * 3/5/9 + 1 -> (add (shYadd (shl X, N), (shl X, N)), X)
19782 if (Shift == 0) {
19783 SDLoc DL(N);
19784 EVT VT = N->getValueType(ResNo: 0);
19785 SDValue X = N->getOperand(Num: 0);
19786 SDValue Shl =
19787 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShX, DL, VT));
19788 SDValue ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: Shl,
19789 N2: DAG.getTargetConstant(Val: ShY, DL, VT), N3: Shl);
19790 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: ShlAdd, N2: X);
19791 }
19792 }
19793 return SDValue();
19794}
19795
19796// Try to expand a scalar multiply to a faster sequence.
19797static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
19798 TargetLowering::DAGCombinerInfo &DCI,
19799 const RISCVSubtarget &Subtarget) {
19800
19801 EVT VT = N->getValueType(ResNo: 0);
19802
19803 // LI + MUL is usually smaller than the alternative sequence.
19804 if (DAG.getMachineFunction().getFunction().hasMinSize())
19805 return SDValue();
19806
19807 if (VT != Subtarget.getXLenVT())
19808 return SDValue();
19809
19810 bool ShouldExpandMul =
19811 (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) ||
19812 !Subtarget.hasStdExtZmmul();
19813 if (!ShouldExpandMul)
19814 return SDValue();
19815
19816 ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
19817 if (!CNode)
19818 return SDValue();
19819 uint64_t MulAmt = CNode->getZExtValue();
19820
19821 // Don't do this if the Xqciac extension is enabled and the MulAmt in simm12.
19822 if (Subtarget.hasVendorXqciac() && isInt<12>(x: CNode->getSExtValue()))
19823 return SDValue();
19824
19825 // WARNING: The code below is knowingly incorrect with regards to undef
19826 // semantics. We're adding additional uses of X here, and in principle, we
19827 // should be freezing X before doing so. However, adding freeze here causes
19828 // real regressions, and no other target properly freezes X in these cases
19829 // either.
19830 if (Subtarget.hasShlAdd(ShAmt: 3)) {
19831 // 3/5/9 * 2^N -> (shl (shXadd X, X), N)
19832 // 3/5/9 * 3/5/9 * 2^N - In particular, this covers multiples
19833 // of 25 which happen to be quite common.
19834 // (2/4/8 * 3/5/9 + 1) * 2^N
19835 unsigned Shift = llvm::countr_zero(Val: MulAmt);
19836 if (SDValue V = expandMulToShlAddShlAdd(N, DAG, MulAmt: MulAmt >> Shift, Shift))
19837 return V;
19838
19839 // If this is a power 2 + 2/4/8, we can use a shift followed by a single
19840 // shXadd. First check if this a sum of two power of 2s because that's
19841 // easy. Then count how many zeros are up to the first bit.
19842 SDValue X = N->getOperand(Num: 0);
19843 if (Shift >= 1 && Shift <= 3 && isPowerOf2_64(Value: MulAmt & (MulAmt - 1))) {
19844 unsigned ShiftAmt = llvm::countr_zero(Val: (MulAmt & (MulAmt - 1)));
19845 SDLoc DL(N);
19846 SDValue Shift1 =
19847 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShiftAmt, DL, VT));
19848 return DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
19849 N2: DAG.getTargetConstant(Val: Shift, DL, VT), N3: Shift1);
19850 }
19851
19852 // TODO: 2^(C1>3) * 3/5/9 - 1
19853
19854 // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
19855 if (MulAmt > 2 && isPowerOf2_64(Value: (MulAmt - 1) & (MulAmt - 2))) {
19856 unsigned ScaleShift = llvm::countr_zero(Val: MulAmt - 1);
19857 if (ScaleShift >= 1 && ScaleShift < 4) {
19858 unsigned ShiftAmt = llvm::countr_zero(Val: (MulAmt - 1) & (MulAmt - 2));
19859 SDLoc DL(N);
19860 SDValue Shift1 =
19861 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShiftAmt, DL, VT));
19862 return DAG.getNode(
19863 Opcode: ISD::ADD, DL, VT, N1: Shift1,
19864 N2: DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
19865 N2: DAG.getTargetConstant(Val: ScaleShift, DL, VT), N3: X));
19866 }
19867 }
19868
19869 // 2^N - 3/5/9 --> (sub (shl X, C1), (shXadd X, x))
19870 for (uint64_t Offset : {3, 5, 9}) {
19871 if (isPowerOf2_64(Value: MulAmt + Offset)) {
19872 unsigned ShAmt = llvm::countr_zero(Val: MulAmt + Offset);
19873 if (ShAmt >= VT.getSizeInBits())
19874 continue;
19875 SDLoc DL(N);
19876 SDValue Shift1 =
19877 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShAmt, DL, VT));
19878 SDValue Mul359 =
19879 DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
19880 N2: DAG.getTargetConstant(Val: Log2_64(Value: Offset - 1), DL, VT), N3: X);
19881 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Shift1, N2: Mul359);
19882 }
19883 }
19884 }
19885
19886 if (SDValue V = expandMulToAddOrSubOfShl(N, DAG, MulAmt))
19887 return V;
19888
19889 if (!Subtarget.hasStdExtZmmul())
19890 return expandMulToNAFSequence(N, DAG, MulAmt);
19891
19892 return SDValue();
19893}
19894
19895// Combine vXi32 (mul (and (lshr X, 15), 0x10001), 0xffff) ->
19896// (bitcast (sra (v2Xi16 (bitcast X)), 15))
19897// Same for other equivalent types with other equivalent constants.
19898static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG) {
19899 EVT VT = N->getValueType(ResNo: 0);
19900 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19901
19902 // Do this for legal vectors unless they are i1 or i8 vectors.
19903 if (!VT.isVector() || !TLI.isTypeLegal(VT) || VT.getScalarSizeInBits() < 16)
19904 return SDValue();
19905
19906 if (N->getOperand(Num: 0).getOpcode() != ISD::AND ||
19907 N->getOperand(Num: 0).getOperand(i: 0).getOpcode() != ISD::SRL)
19908 return SDValue();
19909
19910 SDValue And = N->getOperand(Num: 0);
19911 SDValue Srl = And.getOperand(i: 0);
19912
19913 APInt V1, V2, V3;
19914 if (!ISD::isConstantSplatVector(N: N->getOperand(Num: 1).getNode(), SplatValue&: V1) ||
19915 !ISD::isConstantSplatVector(N: And.getOperand(i: 1).getNode(), SplatValue&: V2) ||
19916 !ISD::isConstantSplatVector(N: Srl.getOperand(i: 1).getNode(), SplatValue&: V3))
19917 return SDValue();
19918
19919 unsigned HalfSize = VT.getScalarSizeInBits() / 2;
19920 if (!V1.isMask(numBits: HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
19921 V3 != (HalfSize - 1))
19922 return SDValue();
19923
19924 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(),
19925 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HalfSize),
19926 EC: VT.getVectorElementCount() * 2);
19927 SDLoc DL(N);
19928 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Srl.getOperand(i: 0));
19929 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT: HalfVT, N1: Cast,
19930 N2: DAG.getConstant(Val: HalfSize - 1, DL, VT: HalfVT));
19931 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Sra);
19932}
19933
19934static SDValue combinePExtWideningMul(SDNode *N, SelectionDAG &DAG,
19935 const RISCVSubtarget &Subtarget) {
19936 if (!Subtarget.hasStdExtP())
19937 return SDValue();
19938
19939 EVT VT = N->getValueType(ResNo: 0);
19940 if (VT != MVT::v4i16 && VT != MVT::v2i32)
19941 return SDValue();
19942
19943 SDValue N0 = N->getOperand(Num: 0);
19944 SDValue N1 = N->getOperand(Num: 1);
19945 bool N0IsSExt = N0.getOpcode() == ISD::SIGN_EXTEND;
19946 bool N0IsZExt = N0.getOpcode() == ISD::ZERO_EXTEND;
19947 bool N1IsSExt = N1.getOpcode() == ISD::SIGN_EXTEND;
19948 bool N1IsZExt = N1.getOpcode() == ISD::ZERO_EXTEND;
19949
19950 if (!(N0IsSExt || N0IsZExt) || !(N1IsSExt || N1IsZExt) || !N0.hasOneUse() ||
19951 !N1.hasOneUse())
19952 return SDValue();
19953
19954 SDValue A = N0.getOperand(i: 0);
19955 SDValue B = N1.getOperand(i: 0);
19956 EVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
19957 if (A.getValueType() != SrcVT || B.getValueType() != SrcVT)
19958 return SDValue();
19959
19960 unsigned RV32Opc, RV64Opc;
19961 bool IsSignedUnsigned = false;
19962 if (N0IsSExt && N1IsSExt) {
19963 RV32Opc = RISCVISD::PWMUL;
19964 RV64Opc = RISCVISD::PMUL_HALVES_01;
19965 } else if (N0IsZExt && N1IsZExt) {
19966 RV32Opc = RISCVISD::PWMULU;
19967 RV64Opc = RISCVISD::PMULU_HALVES_01;
19968 } else {
19969 IsSignedUnsigned = true;
19970 RV32Opc = RISCVISD::PWMULSU;
19971 RV64Opc = RISCVISD::PMULSU_HALVES_00;
19972 if (N0IsZExt && N1IsSExt)
19973 std::swap(a&: A, b&: B);
19974 }
19975
19976 SDLoc DL(N);
19977 if (!Subtarget.is64Bit())
19978 return DAG.getNode(Opcode: RV32Opc, DL, VT, N1: A, N2: B);
19979
19980 MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
19981 A = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: LegalSrcVT, N1: A, N2: DAG.getUNDEF(VT: SrcVT));
19982 B = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: LegalSrcVT, N1: B, N2: DAG.getUNDEF(VT: SrcVT));
19983
19984 if (IsSignedUnsigned) {
19985 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: LegalSrcVT);
19986 A = DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT: LegalSrcVT, N1: A, N2: Zero);
19987 B = DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT: LegalSrcVT, N1: B, N2: Zero);
19988 return DAG.getNode(Opcode: RV64Opc, DL, VT, N1: A, N2: B);
19989 }
19990
19991 SDValue Zip = DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT: LegalSrcVT, N1: A, N2: B);
19992 return DAG.getNode(Opcode: RV64Opc, DL, VT, N1: Zip, N2: Zip);
19993}
19994
19995static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
19996 TargetLowering::DAGCombinerInfo &DCI,
19997 const RISCVSubtarget &Subtarget) {
19998 EVT VT = N->getValueType(ResNo: 0);
19999 if (!VT.isVector())
20000 return expandMul(N, DAG, DCI, Subtarget);
20001
20002 SDLoc DL(N);
20003 SDValue N0 = N->getOperand(Num: 0);
20004 SDValue N1 = N->getOperand(Num: 1);
20005 SDValue MulOper;
20006 unsigned AddSubOpc;
20007
20008 // vmadd: (mul (add x, 1), y) -> (add (mul x, y), y)
20009 // (mul x, add (y, 1)) -> (add x, (mul x, y))
20010 // vnmsub: (mul (sub 1, x), y) -> (sub y, (mul x, y))
20011 // (mul x, (sub 1, y)) -> (sub x, (mul x, y))
20012 auto IsAddSubWith1 = [&](SDValue V) -> bool {
20013 AddSubOpc = V->getOpcode();
20014 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
20015 SDValue Opnd = V->getOperand(Num: 1);
20016 MulOper = V->getOperand(Num: 0);
20017 if (AddSubOpc == ISD::SUB)
20018 std::swap(a&: Opnd, b&: MulOper);
20019 if (isOneOrOneSplat(V: Opnd))
20020 return true;
20021 }
20022 return false;
20023 };
20024
20025 if (IsAddSubWith1(N0)) {
20026 SDValue MulVal = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1, N2: MulOper);
20027 return DAG.getNode(Opcode: AddSubOpc, DL, VT, N1, N2: MulVal);
20028 }
20029
20030 if (IsAddSubWith1(N1)) {
20031 SDValue MulVal = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N0, N2: MulOper);
20032 return DAG.getNode(Opcode: AddSubOpc, DL, VT, N1: N0, N2: MulVal);
20033 }
20034
20035 if (SDValue V = combineBinOpOfZExt(N, DAG))
20036 return V;
20037
20038 if (SDValue V = combinePExtWideningMul(N, DAG, Subtarget))
20039 return V;
20040
20041 if (SDValue V = combineVectorMulToSraBitcast(N, DAG))
20042 return V;
20043
20044 return SDValue();
20045}
20046
20047/// According to the property that indexed load/store instructions zero-extend
20048/// their indices, try to narrow the type of index operand.
20049static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
20050 if (isIndexTypeSigned(IndexType))
20051 return false;
20052
20053 if (!N->hasOneUse())
20054 return false;
20055
20056 EVT VT = N.getValueType();
20057 SDLoc DL(N);
20058
20059 // In general, what we're doing here is seeing if we can sink a truncate to
20060 // a smaller element type into the expression tree building our index.
20061 // TODO: We can generalize this and handle a bunch more cases if useful.
20062
20063 // Narrow a buildvector to the narrowest element type. This requires less
20064 // work and less register pressure at high LMUL, and creates smaller constants
20065 // which may be cheaper to materialize.
20066 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode())) {
20067 KnownBits Known = DAG.computeKnownBits(Op: N);
20068 unsigned ActiveBits = std::max(a: 8u, b: Known.countMaxActiveBits());
20069 LLVMContext &C = *DAG.getContext();
20070 EVT ResultVT = EVT::getIntegerVT(Context&: C, BitWidth: ActiveBits).getRoundIntegerType(Context&: C);
20071 if (ResultVT.bitsLT(VT: VT.getVectorElementType())) {
20072 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL,
20073 VT: VT.changeVectorElementType(Context&: C, EltVT: ResultVT), Operand: N);
20074 return true;
20075 }
20076 }
20077
20078 // Handle the pattern (shl (zext x to ty), C) and bits(x) + C < bits(ty).
20079 if (N.getOpcode() != ISD::SHL)
20080 return false;
20081
20082 SDValue N0 = N.getOperand(i: 0);
20083 if (N0.getOpcode() != ISD::ZERO_EXTEND &&
20084 N0.getOpcode() != RISCVISD::VZEXT_VL)
20085 return false;
20086 if (!N0->hasOneUse())
20087 return false;
20088
20089 APInt ShAmt;
20090 SDValue N1 = N.getOperand(i: 1);
20091 if (!ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: ShAmt))
20092 return false;
20093
20094 SDValue Src = N0.getOperand(i: 0);
20095 EVT SrcVT = Src.getValueType();
20096 unsigned SrcElen = SrcVT.getScalarSizeInBits();
20097
20098 // Consider any leading zeros in the source.
20099 SrcElen -= DAG.computeKnownBits(Op: Src).countMinLeadingZeros();
20100
20101 unsigned ShAmtV = ShAmt.getZExtValue();
20102 unsigned NewElen = PowerOf2Ceil(A: SrcElen + ShAmtV);
20103 NewElen = std::max(a: NewElen, b: 8U);
20104 // Make sure the new elen is at least as large as the original elen.
20105 NewElen = std::max<unsigned>(a: NewElen, b: SrcVT.getScalarSizeInBits());
20106
20107 // Skip if NewElen is not narrower than the original extended type.
20108 if (NewElen >= N0.getValueType().getScalarSizeInBits())
20109 return false;
20110
20111 EVT NewEltVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewElen);
20112 EVT NewVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: NewEltVT);
20113
20114 SDValue NewExt = DAG.getNode(Opcode: N0->getOpcode(), DL, VT: NewVT, Ops: N0->ops());
20115 SDValue NewShAmtVec = DAG.getConstant(Val: ShAmtV, DL, VT: NewVT);
20116 N = DAG.getNode(Opcode: ISD::SHL, DL, VT: NewVT, N1: NewExt, N2: NewShAmtVec);
20117 return true;
20118}
20119
20120/// Given
20121/// ```
20122/// %b = splat %base
20123/// %s = <%offset, %offset + 1, %offset + 2, %offset + 3, ...>
20124/// %a = add nuw %b, %s
20125/// %N = splat %n
20126/// %p = setcc ult %a, %N
20127/// ```
20128/// in which the SETCC's condition code could be (U)LT or (U)GT.
20129/// We can turn it into
20130/// ```
20131/// %s = <0, 1, 2, 3, ...>
20132/// %m = sub %n, min(%n, %base + %offset)
20133/// %M = splat %m
20134/// %p = setcc ult %s, %M
20135/// ```
20136/// where the SETCC's condition code is also canonicalized into (U)LT;
20137/// The idea behind this canonicalization is that if %p is used as a mask
20138/// in a masked.load/store, we can easily turn it to use VL-predicate later
20139/// by assigning `vl = min(%m, <number of vector elements>)`.
20140static SDValue canonicalizeMaskForVLPredicate(
20141 EVT MaskVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL,
20142 SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI) {
20143
20144 // This transformation performs checks against the vector element type, which
20145 // is also used to generate scalar value that will be splatted later. Because
20146 // of this, the emitted scalar value might not be legal type and therefore we
20147 // need to run this before type legalizer.
20148 if (!DCI.isBeforeLegalize())
20149 return SDValue();
20150
20151 using namespace SDPatternMatch;
20152 if (!MaskVT.isFixedLengthVector() ||
20153 !(CC == ISD::SETUGT || CC == ISD::SETGT || CC == ISD::SETULT ||
20154 CC == ISD::SETLT) ||
20155 !LHS.getValueType().isInteger())
20156 return SDValue();
20157
20158 // Canonicalize the comparison.
20159 if (CC == ISD::SETUGT || CC == ISD::SETGT) {
20160 std::swap(a&: LHS, b&: RHS);
20161 CC = ISD::getSetCCSwappedOperands(Operation: CC);
20162 }
20163 bool IsSigned = ISD::isSignedIntSetCC(Code: CC);
20164
20165 EVT ElementVT = LHS.getValueType().getScalarType();
20166 uint64_t ElementSize = LHS.getScalarValueSizeInBits();
20167
20168 SDValue Boundary = DAG.getSplatValue(V: RHS);
20169 if (!Boundary)
20170 return SDValue();
20171
20172 SDValue LHSOp0, LHSOp1;
20173 if (IsSigned) {
20174 if (!sd_match(N: LHS, P: m_NSWAddLike(L: m_Value(N&: LHSOp0), R: m_Value(N&: LHSOp1))))
20175 return SDValue();
20176 } else {
20177 if (!sd_match(N: LHS, P: m_NUWAddLike(L: m_Value(N&: LHSOp0), R: m_Value(N&: LHSOp1))))
20178 return SDValue();
20179 }
20180
20181 SDValue BaseIndex = DAG.getSplatValue(V: LHSOp0);
20182 if (!BaseIndex) {
20183 std::swap(a&: LHSOp0, b&: LHSOp1);
20184 BaseIndex = DAG.getSplatValue(V: LHSOp0);
20185 }
20186 if (!BaseIndex || !isa<BuildVectorSDNode>(Val: LHSOp1))
20187 return SDValue();
20188
20189 // Return {a,n} from a build_vector sequence of {a, a+n, a+2n, a+3n, ....}
20190 auto StepVector = cast<BuildVectorSDNode>(Val&: LHSOp1)->isArithmeticSequence();
20191 if (!StepVector || !StepVector->second.isOne())
20192 return SDValue();
20193 const APInt &Start = StepVector->first;
20194
20195 Boundary = DAG.getExtOrTrunc(IsSigned, Op: Boundary, DL, VT: ElementVT);
20196 BaseIndex = DAG.getExtOrTrunc(IsSigned, Op: BaseIndex, DL, VT: ElementVT);
20197 SDValue Offset = DAG.getConstant(Val: IsSigned ? Start.sextOrTrunc(width: ElementSize)
20198 : Start.zextOrTrunc(width: ElementSize),
20199 DL, VT: ElementVT);
20200
20201 if (IsSigned) {
20202 // Two additional conditions:
20203 // 1. Offset + BaseIndex never overflow
20204 if (!DAG.willNotOverflowAdd(/*IsSigned=*/true, N0: BaseIndex, N1: Offset))
20205 return SDValue();
20206
20207 // 2. Offset + BaseIndex has to be non-negative
20208 auto BaseIndexKB = DAG.computeKnownBits(Op: BaseIndex);
20209 auto OffsetKB = DAG.computeKnownBits(Op: Offset);
20210 if (!KnownBits::add(LHS: BaseIndexKB, RHS: OffsetKB, /*NSW=*/true,
20211 /*NUW=*/false)
20212 .isNonNegative())
20213 return SDValue();
20214 }
20215
20216 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
20217 SDValue NewStepVector = DAG.getStepVector(DL, ResVT: LHS.getValueType());
20218 BaseIndex = DAG.getNode(Opcode: ISD::ADD, DL, VT: ElementVT, N1: BaseIndex, N2: Offset,
20219 Flags: IsSigned ? SDNodeFlags::NoSignedWrap
20220 : SDNodeFlags::NoUnsignedWrap);
20221 BaseIndex = DAG.getNode(Opcode: MinOpc, DL, VT: ElementVT, N1: Boundary, N2: BaseIndex);
20222 Boundary = DAG.getNode(Opcode: ISD::SUB, DL, VT: ElementVT, N1: Boundary, N2: BaseIndex);
20223 Boundary = DAG.getSplat(VT: RHS.getValueType(), DL, Op: Boundary);
20224
20225 return DAG.getSetCC(DL, VT: MaskVT, LHS: NewStepVector, RHS: Boundary, Cond: CC);
20226}
20227
20228/// Try to map an integer comparison with size > XLEN to vector instructions
20229/// before type legalization splits it up into chunks.
20230static SDValue
20231combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
20232 const SDLoc &DL, SelectionDAG &DAG,
20233 const RISCVSubtarget &Subtarget) {
20234 assert(ISD::isIntEqualitySetCC(CC) && "Bad comparison predicate");
20235
20236 if (!Subtarget.useRVVForFixedLengthVectors())
20237 return SDValue();
20238
20239 MVT XLenVT = Subtarget.getXLenVT();
20240 EVT OpVT = X.getValueType();
20241 // We're looking for an oversized integer equality comparison.
20242 if (!OpVT.isScalarInteger())
20243 return SDValue();
20244
20245 unsigned OpSize = OpVT.getSizeInBits();
20246 // The size should be larger than XLen and smaller than the maximum vector
20247 // size.
20248 if (OpSize <= Subtarget.getXLen() ||
20249 OpSize > Subtarget.getRealMinVLen() *
20250 Subtarget.getMaxLMULForFixedLengthVectors())
20251 return SDValue();
20252
20253 // Don't perform this combine if constructing the vector will be expensive.
20254 auto IsVectorBitCastCheap = [](SDValue X) {
20255 X = peekThroughBitcasts(V: X);
20256 return isa<ConstantSDNode>(Val: X) || X.getValueType().isVector() ||
20257 X.getOpcode() == ISD::LOAD;
20258 };
20259 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
20260 return SDValue();
20261
20262 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
20263 Kind: Attribute::NoImplicitFloat))
20264 return SDValue();
20265
20266 // Bail out for non-byte-sized types.
20267 if (!OpVT.isByteSized())
20268 return SDValue();
20269
20270 // Find a preferred vector element type by inspecting how the value is used.
20271 auto GetPreferredEltVT = [](SDValue V) -> MVT {
20272 // Look backwards: check if V itself is derived from a vector
20273 SDValue Peek = peekThroughBitcasts(V);
20274 EVT PeekVT = Peek.getValueType();
20275
20276 if (PeekVT.isVector() && PeekVT.isInteger()) {
20277 EVT EltVT = PeekVT.getVectorElementType();
20278 if (EltVT.isSimple())
20279 return EltVT.getSimpleVT();
20280 }
20281
20282 // Look forwards: check if V is bitcasted to a vector elsewhere in the DAG
20283 for (SDUse &Use : V->uses()) {
20284 // Ensure we are checking the use of the specific value result, not the
20285 // node's chain
20286 if (Use.getResNo() != V.getResNo())
20287 continue;
20288
20289 SDNode *User = Use.getUser();
20290 if (User->getOpcode() == ISD::BITCAST) {
20291 EVT CastVT = User->getValueType(ResNo: 0);
20292 if (CastVT.isVector() && CastVT.isInteger()) {
20293 EVT EltVT = CastVT.getVectorElementType();
20294 if (EltVT.isSimple())
20295 return EltVT.getSimpleVT();
20296 }
20297 }
20298 }
20299 return MVT::INVALID_SIMPLE_VALUE_TYPE;
20300 };
20301
20302 auto IsValidEltVT = [&](MVT VT) {
20303 if (VT == MVT::INVALID_SIMPLE_VALUE_TYPE || !VT.isInteger())
20304 return false;
20305
20306 // Make sure we don't try to create an impossible vector type where the
20307 // elements don't perfectly fill up the OpSize boundary.
20308 unsigned EltSize = VT.getSizeInBits();
20309 if (OpSize % EltSize != 0)
20310 return false;
20311
20312 // Construct the proposed vector type to check its legality
20313 unsigned NumElts = OpSize / EltSize;
20314 EVT TestVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NumElts);
20315 return Subtarget.getTargetLowering()->isTypeLegal(VT: TestVecVT);
20316 };
20317
20318 // Get preferred VT from either X or Y.
20319 MVT EltVT = GetPreferredEltVT(X);
20320 if (!IsValidEltVT(EltVT))
20321 EltVT = GetPreferredEltVT(Y);
20322
20323 // If both are unsuitable, use the safe default (i8)
20324 if (!IsValidEltVT(EltVT))
20325 EltVT = MVT::i8;
20326
20327 unsigned EltSize = EltVT.getSizeInBits();
20328 unsigned NumElts = OpSize / EltSize;
20329 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
20330 EVT CmpVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: NumElts);
20331
20332 SDValue VecX = DAG.getBitcast(VT: VecVT, V: X);
20333 SDValue VecY;
20334 // Constant fold the common case of comparing with zero. Later optimizations
20335 // might not do this for us.
20336 if (isNullConstant(V: Y))
20337 VecY = DAG.getConstant(Val: 0, DL, VT: VecVT);
20338 else
20339 VecY = DAG.getBitcast(VT: VecVT, V: Y);
20340
20341 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: VecX, RHS: VecY, Cond: ISD::SETNE);
20342 return DAG.getSetCC(DL, VT, LHS: DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: XLenVT, Operand: Cmp),
20343 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: CC);
20344}
20345
20346static SDValue performSETCCCombine(SDNode *N,
20347 TargetLowering::DAGCombinerInfo &DCI,
20348 const RISCVSubtarget &Subtarget) {
20349 SelectionDAG &DAG = DCI.DAG;
20350 SDLoc dl(N);
20351 SDValue N0 = N->getOperand(Num: 0);
20352 SDValue N1 = N->getOperand(Num: 1);
20353 EVT VT = N->getValueType(ResNo: 0);
20354 EVT OpVT = N0.getValueType();
20355
20356 ISD::CondCode Cond = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
20357 if (SDValue V =
20358 canonicalizeMaskForVLPredicate(MaskVT: VT, LHS: N0, RHS: N1, CC: Cond, DL: dl, DAG, DCI))
20359 return V;
20360
20361 // Looking for an equality compare.
20362 if (!isIntEqualitySetCC(Code: Cond))
20363 return SDValue();
20364
20365 if (SDValue V =
20366 combineVectorSizedSetCCEquality(VT, X: N0, Y: N1, CC: Cond, DL: dl, DAG, Subtarget))
20367 return V;
20368
20369 if (DCI.isAfterLegalizeDAG() && isa<ConstantSDNode>(Val: N1) &&
20370 N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
20371 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
20372 const APInt &AndRHSC = N0.getConstantOperandAPInt(i: 1);
20373 // (X & -(1 << C)) == 0 -> (X >> C) == 0 if the AND constant can't use ANDI.
20374 if (isNullConstant(V: N1) && !isInt<12>(x: AndRHSC.getSExtValue()) &&
20375 AndRHSC.isNegatedPowerOf2()) {
20376 unsigned ShiftBits = AndRHSC.countr_zero();
20377 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: N0.getOperand(i: 0),
20378 N2: DAG.getConstant(Val: ShiftBits, DL: dl, VT: OpVT));
20379 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: N1, Cond);
20380 }
20381
20382 // Similar to above but handling the lower 32 bits by using sraiw. Allow
20383 // comparing with constants other than 0 if the constant can be folded into
20384 // addi or xori after shifting.
20385 uint64_t N1Int = cast<ConstantSDNode>(Val&: N1)->getZExtValue();
20386 uint64_t AndRHSInt = AndRHSC.getZExtValue();
20387 if (OpVT == MVT::i64 && isUInt<32>(x: AndRHSInt) &&
20388 isPowerOf2_32(Value: -uint32_t(AndRHSInt)) && (N1Int & AndRHSInt) == N1Int) {
20389 unsigned ShiftBits = llvm::countr_zero(Val: AndRHSInt);
20390 int64_t NewC = SignExtend64<32>(x: N1Int) >> ShiftBits;
20391 if (ShiftBits != 0 && NewC >= -2048 && NewC <= 2048) {
20392 SDValue SExt =
20393 DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OpVT, N1: N0.getOperand(i: 0),
20394 N2: DAG.getValueType(MVT::i32));
20395 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: OpVT, N1: SExt,
20396 N2: DAG.getConstant(Val: ShiftBits, DL: dl, VT: OpVT));
20397 return DAG.getSetCC(DL: dl, VT, LHS: Shift,
20398 RHS: DAG.getSignedConstant(Val: NewC, DL: dl, VT: OpVT), Cond);
20399 }
20400 }
20401
20402 // Fold (and X, Mask) ==/!= C -> X ==/!= sext(C, countr_one(Mask)) if the
20403 // Mask is only clearing redundant sign bits.
20404 if (isMask_64(Value: AndRHSInt)) {
20405 unsigned TrailingOnes = llvm::countr_one(Value: AndRHSInt);
20406 unsigned N1Width = llvm::bit_width(Value: N1Int);
20407 int64_t N1SExt = SignExtend64(X: N1Int, B: TrailingOnes);
20408 if (N1Width <= TrailingOnes && isInt<12>(x: N1SExt) &&
20409 DAG.ComputeMaxSignificantBits(Op: N0.getOperand(i: 0)) <= TrailingOnes)
20410 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
20411 RHS: DAG.getSignedConstant(Val: N1SExt, DL: dl, VT: OpVT), Cond);
20412 }
20413 }
20414
20415 // Replace (seteq (i64 (and X, 0xffffffff)), C1) with
20416 // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
20417 // bit 31. Same for setne. C1' may be cheaper to materialize and the
20418 // sext_inreg can become a sext.w instead of a shift pair.
20419 if (OpVT != MVT::i64 || !Subtarget.is64Bit())
20420 return SDValue();
20421
20422 // RHS needs to be a constant.
20423 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
20424 if (!N1C)
20425 return SDValue();
20426
20427 // LHS needs to be (and X, 0xffffffff).
20428 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
20429 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) ||
20430 N0.getConstantOperandVal(i: 1) != UINT64_C(0xffffffff))
20431 return SDValue();
20432
20433 // Don't do this if the sign bit is provably zero, it will be turned back into
20434 // an AND.
20435 APInt SignMask = APInt::getOneBitSet(numBits: 64, BitNo: 31);
20436 if (DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask: SignMask))
20437 return SDValue();
20438
20439 const APInt &C1 = N1C->getAPIntValue();
20440
20441 // If the constant is larger than 2^32 - 1 it is impossible for both sides
20442 // to be equal.
20443 if (C1.getActiveBits() > 32)
20444 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
20445
20446 SDValue SExtOp = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: N, VT: OpVT,
20447 N1: N0.getOperand(i: 0), N2: DAG.getValueType(MVT::i32));
20448 return DAG.getSetCC(DL: dl, VT, LHS: SExtOp, RHS: DAG.getConstant(Val: C1.trunc(width: 32).sext(width: 64),
20449 DL: dl, VT: OpVT), Cond);
20450}
20451
20452static SDValue
20453performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
20454 const RISCVSubtarget &Subtarget) {
20455 SelectionDAG &DAG = DCI.DAG;
20456 SDValue Src = N->getOperand(Num: 0);
20457 EVT VT = N->getValueType(ResNo: 0);
20458 EVT SrcVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
20459 unsigned Opc = Src.getOpcode();
20460 SDLoc DL(N);
20461
20462 // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
20463 // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
20464 if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(VT: MVT::i16) &&
20465 Subtarget.hasStdExtZfhmin())
20466 return DAG.getNode(Opcode: RISCVISD::FMV_X_SIGNEXTH, DL, VT, Operand: Src.getOperand(i: 0));
20467
20468 // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
20469 if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
20470 VT == MVT::i64 && !isa<ConstantSDNode>(Val: Src.getOperand(i: 1)) &&
20471 DAG.computeKnownBits(Op: Src.getOperand(i: 1)).countMaxActiveBits() <= 5)
20472 return DAG.getNode(Opcode: RISCVISD::SLLW, DL, VT, N1: Src.getOperand(i: 0),
20473 N2: Src.getOperand(i: 1));
20474
20475 // Fold (sext_inreg (setcc), i1) -> (sub 0, (setcc))
20476 if (Opc == ISD::SETCC && SrcVT == MVT::i1 && DCI.isAfterLegalizeDAG())
20477 return DAG.getNegative(Val: Src, DL, VT);
20478
20479 // Fold (sext_inreg (xor (setcc), -1), i1) -> (add (setcc), -1)
20480 if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
20481 isAllOnesConstant(V: Src.getOperand(i: 1)) &&
20482 Src.getOperand(i: 0).getOpcode() == ISD::SETCC && DCI.isAfterLegalizeDAG())
20483 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Src.getOperand(i: 0),
20484 N2: DAG.getAllOnesConstant(DL, VT));
20485
20486 return SDValue();
20487}
20488
20489namespace {
20490// Forward declaration of the structure holding the necessary information to
20491// apply a combine.
20492struct CombineResult;
20493
20494enum ExtKind : uint8_t {
20495 ZExt = 1 << 0,
20496 SExt = 1 << 1,
20497 FPExt = 1 << 2,
20498 BF16Ext = 1 << 3
20499};
20500/// Helper class for folding sign/zero extensions.
20501/// In particular, this class is used for the following combines:
20502/// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
20503/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
20504/// mul | mul_vl -> vwmul(u) | vwmul_su
20505/// shl | shl_vl -> vwsll
20506/// fadd -> vfwadd | vfwadd_w
20507/// fsub -> vfwsub | vfwsub_w
20508/// fmul -> vfwmul
20509/// An object of this class represents an operand of the operation we want to
20510/// combine.
20511/// E.g., when trying to combine `mul_vl a, b`, we will have one instance of
20512/// NodeExtensionHelper for `a` and one for `b`.
20513///
20514/// This class abstracts away how the extension is materialized and
20515/// how its number of users affect the combines.
20516///
20517/// In particular:
20518/// - VWADD_W is conceptually == add(op0, sext(op1))
20519/// - VWADDU_W == add(op0, zext(op1))
20520/// - VWSUB_W == sub(op0, sext(op1))
20521/// - VWSUBU_W == sub(op0, zext(op1))
20522/// - VFWADD_W == fadd(op0, fpext(op1))
20523/// - VFWSUB_W == fsub(op0, fpext(op1))
20524/// And VMV_V_X_VL, depending on the value, is conceptually equivalent to
20525/// zext|sext(smaller_value).
20526struct NodeExtensionHelper {
20527 /// Records if this operand is like being zero extended.
20528 bool SupportsZExt;
20529 /// Records if this operand is like being sign extended.
20530 /// Note: SupportsZExt and SupportsSExt are not mutually exclusive. For
20531 /// instance, a splat constant (e.g., 3), would support being both sign and
20532 /// zero extended.
20533 bool SupportsSExt;
20534 /// Records if this operand is like being floating point extended.
20535 bool SupportsFPExt;
20536 /// Records if this operand is extended from bf16.
20537 bool SupportsBF16Ext;
20538 /// This boolean captures whether we care if this operand would still be
20539 /// around after the folding happens.
20540 bool EnforceOneUse;
20541 /// Original value that this NodeExtensionHelper represents.
20542 SDValue OrigOperand;
20543
20544 /// Get the value feeding the extension or the value itself.
20545 /// E.g., for zext(a), this would return a.
20546 SDValue getSource() const {
20547 switch (OrigOperand.getOpcode()) {
20548 case ISD::ZERO_EXTEND:
20549 case ISD::SIGN_EXTEND:
20550 case RISCVISD::VSEXT_VL:
20551 case RISCVISD::VZEXT_VL:
20552 case RISCVISD::FP_EXTEND_VL:
20553 return OrigOperand.getOperand(i: 0);
20554 default:
20555 return OrigOperand;
20556 }
20557 }
20558
20559 /// Check if this instance represents a splat.
20560 bool isSplat() const {
20561 return OrigOperand.getOpcode() == RISCVISD::VMV_V_X_VL ||
20562 OrigOperand.getOpcode() == ISD::SPLAT_VECTOR;
20563 }
20564
20565 /// Get the extended opcode.
20566 unsigned getExtOpc(ExtKind SupportsExt) const {
20567 switch (SupportsExt) {
20568 case ExtKind::SExt:
20569 return RISCVISD::VSEXT_VL;
20570 case ExtKind::ZExt:
20571 return RISCVISD::VZEXT_VL;
20572 case ExtKind::FPExt:
20573 case ExtKind::BF16Ext:
20574 return RISCVISD::FP_EXTEND_VL;
20575 }
20576 llvm_unreachable("Unknown ExtKind enum");
20577 }
20578
20579 /// Get or create a value that can feed \p Root with the given extension \p
20580 /// SupportsExt. If \p SExt is std::nullopt, this returns the source of this
20581 /// operand. \see ::getSource().
20582 SDValue getOrCreateExtendedOp(SDNode *Root, SelectionDAG &DAG,
20583 const RISCVSubtarget &Subtarget,
20584 std::optional<ExtKind> SupportsExt) const {
20585 if (!SupportsExt.has_value())
20586 return OrigOperand;
20587
20588 MVT NarrowVT = getNarrowType(Root, SupportsExt: *SupportsExt);
20589
20590 SDValue Source = getSource();
20591 assert(Subtarget.getTargetLowering()->isTypeLegal(Source.getValueType()));
20592 if (Source.getValueType() == NarrowVT)
20593 return Source;
20594
20595 unsigned ExtOpc = getExtOpc(SupportsExt: *SupportsExt);
20596
20597 // If we need an extension, we should be changing the type.
20598 SDLoc DL(OrigOperand);
20599 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
20600 switch (OrigOperand.getOpcode()) {
20601 case ISD::ZERO_EXTEND:
20602 case ISD::SIGN_EXTEND:
20603 case RISCVISD::VSEXT_VL:
20604 case RISCVISD::VZEXT_VL:
20605 case RISCVISD::FP_EXTEND_VL:
20606 return DAG.getNode(Opcode: ExtOpc, DL, VT: NarrowVT, N1: Source, N2: Mask, N3: VL);
20607 case ISD::SPLAT_VECTOR:
20608 return DAG.getSplat(VT: NarrowVT, DL, Op: Source.getOperand(i: 0));
20609 case RISCVISD::VMV_V_X_VL:
20610 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: NarrowVT,
20611 N1: DAG.getUNDEF(VT: NarrowVT), N2: Source.getOperand(i: 1), N3: VL);
20612 case RISCVISD::VFMV_V_F_VL:
20613 Source = Source.getOperand(i: 1);
20614 assert(Source.getOpcode() == ISD::FP_EXTEND && "Unexpected source");
20615 Source = Source.getOperand(i: 0);
20616 assert(Source.getValueType() == NarrowVT.getVectorElementType());
20617 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: NarrowVT,
20618 N1: DAG.getUNDEF(VT: NarrowVT), N2: Source, N3: VL);
20619 default:
20620 // Other opcodes can only come from the original LHS of VW(ADD|SUB)_W_VL
20621 // and that operand should already have the right NarrowVT so no
20622 // extension should be required at this point.
20623 llvm_unreachable("Unsupported opcode");
20624 }
20625 }
20626
20627 /// Helper function to get the narrow type for \p Root.
20628 /// The narrow type is the type of \p Root where we divided the size of each
20629 /// element by 2. E.g., if Root's type <2xi16> -> narrow type <2xi8>.
20630 /// \pre Both the narrow type and the original type should be legal.
20631 static MVT getNarrowType(const SDNode *Root, ExtKind SupportsExt) {
20632 MVT VT = Root->getSimpleValueType(ResNo: 0);
20633
20634 // Determine the narrow size.
20635 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
20636
20637 MVT EltVT = SupportsExt == ExtKind::BF16Ext ? MVT::bf16
20638 : SupportsExt == ExtKind::FPExt
20639 ? MVT::getFloatingPointVT(BitWidth: NarrowSize)
20640 : MVT::getIntegerVT(BitWidth: NarrowSize);
20641
20642 assert((int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
20643 "Trying to extend something we can't represent");
20644 MVT NarrowVT = MVT::getVectorVT(VT: EltVT, EC: VT.getVectorElementCount());
20645 return NarrowVT;
20646 }
20647
20648 /// Get the opcode to materialize:
20649 /// Opcode(sext(a), sext(b)) -> newOpcode(a, b)
20650 static unsigned getSExtOpcode(unsigned Opcode) {
20651 switch (Opcode) {
20652 case ISD::ADD:
20653 case RISCVISD::ADD_VL:
20654 case RISCVISD::VWADD_W_VL:
20655 case RISCVISD::VWADDU_W_VL:
20656 case ISD::OR:
20657 case RISCVISD::OR_VL:
20658 return RISCVISD::VWADD_VL;
20659 case ISD::SUB:
20660 case RISCVISD::SUB_VL:
20661 case RISCVISD::VWSUB_W_VL:
20662 case RISCVISD::VWSUBU_W_VL:
20663 return RISCVISD::VWSUB_VL;
20664 case ISD::MUL:
20665 case RISCVISD::MUL_VL:
20666 return RISCVISD::VWMUL_VL;
20667 default:
20668 llvm_unreachable("Unexpected opcode");
20669 }
20670 }
20671
20672 /// Get the opcode to materialize:
20673 /// Opcode(zext(a), zext(b)) -> newOpcode(a, b)
20674 static unsigned getZExtOpcode(unsigned Opcode) {
20675 switch (Opcode) {
20676 case ISD::ADD:
20677 case RISCVISD::ADD_VL:
20678 case RISCVISD::VWADD_W_VL:
20679 case RISCVISD::VWADDU_W_VL:
20680 case ISD::OR:
20681 case RISCVISD::OR_VL:
20682 return RISCVISD::VWADDU_VL;
20683 case ISD::SUB:
20684 case RISCVISD::SUB_VL:
20685 case RISCVISD::VWSUB_W_VL:
20686 case RISCVISD::VWSUBU_W_VL:
20687 return RISCVISD::VWSUBU_VL;
20688 case ISD::MUL:
20689 case RISCVISD::MUL_VL:
20690 return RISCVISD::VWMULU_VL;
20691 case ISD::SHL:
20692 case RISCVISD::SHL_VL:
20693 return RISCVISD::VWSLL_VL;
20694 default:
20695 llvm_unreachable("Unexpected opcode");
20696 }
20697 }
20698
20699 /// Get the opcode to materialize:
20700 /// Opcode(fpext(a), fpext(b)) -> newOpcode(a, b)
20701 static unsigned getFPExtOpcode(unsigned Opcode) {
20702 switch (Opcode) {
20703 case RISCVISD::FADD_VL:
20704 case RISCVISD::VFWADD_W_VL:
20705 return RISCVISD::VFWADD_VL;
20706 case RISCVISD::FSUB_VL:
20707 case RISCVISD::VFWSUB_W_VL:
20708 return RISCVISD::VFWSUB_VL;
20709 case RISCVISD::FMUL_VL:
20710 return RISCVISD::VFWMUL_VL;
20711 case RISCVISD::VFMADD_VL:
20712 return RISCVISD::VFWMADD_VL;
20713 case RISCVISD::VFMSUB_VL:
20714 return RISCVISD::VFWMSUB_VL;
20715 case RISCVISD::VFNMADD_VL:
20716 return RISCVISD::VFWNMADD_VL;
20717 case RISCVISD::VFNMSUB_VL:
20718 return RISCVISD::VFWNMSUB_VL;
20719 default:
20720 llvm_unreachable("Unexpected opcode");
20721 }
20722 }
20723
20724 /// Get the opcode to materialize \p Opcode(sext(a), zext(b)) ->
20725 /// newOpcode(a, b).
20726 static unsigned getSUOpcode(unsigned Opcode) {
20727 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) &&
20728 "SU is only supported for MUL");
20729 return RISCVISD::VWMULSU_VL;
20730 }
20731
20732 /// Get the opcode to materialize
20733 /// \p Opcode(a, s|z|fpext(b)) -> newOpcode(a, b).
20734 static unsigned getWOpcode(unsigned Opcode, ExtKind SupportsExt) {
20735 switch (Opcode) {
20736 case ISD::ADD:
20737 case RISCVISD::ADD_VL:
20738 case ISD::OR:
20739 case RISCVISD::OR_VL:
20740 return SupportsExt == ExtKind::SExt ? RISCVISD::VWADD_W_VL
20741 : RISCVISD::VWADDU_W_VL;
20742 case ISD::SUB:
20743 case RISCVISD::SUB_VL:
20744 return SupportsExt == ExtKind::SExt ? RISCVISD::VWSUB_W_VL
20745 : RISCVISD::VWSUBU_W_VL;
20746 case RISCVISD::FADD_VL:
20747 return RISCVISD::VFWADD_W_VL;
20748 case RISCVISD::FSUB_VL:
20749 return RISCVISD::VFWSUB_W_VL;
20750 default:
20751 llvm_unreachable("Unexpected opcode");
20752 }
20753 }
20754
20755 using CombineToTry = std::function<std::optional<CombineResult>(
20756 SDNode * /*Root*/, const NodeExtensionHelper & /*LHS*/,
20757 const NodeExtensionHelper & /*RHS*/, SelectionDAG &,
20758 const RISCVSubtarget &)>;
20759
20760 /// Check if this node needs to be fully folded or extended for all users.
20761 bool needToPromoteOtherUsers() const { return EnforceOneUse; }
20762
20763 void fillUpExtensionSupportForSplat(SDNode *Root, SelectionDAG &DAG,
20764 const RISCVSubtarget &Subtarget) {
20765 unsigned Opc = OrigOperand.getOpcode();
20766 MVT VT = OrigOperand.getSimpleValueType();
20767
20768 assert((Opc == ISD::SPLAT_VECTOR || Opc == RISCVISD::VMV_V_X_VL) &&
20769 "Unexpected Opcode");
20770
20771 // The pasthru must be undef for tail agnostic.
20772 if (Opc == RISCVISD::VMV_V_X_VL && !OrigOperand.getOperand(i: 0).isUndef())
20773 return;
20774
20775 // Get the scalar value.
20776 SDValue Op = Opc == ISD::SPLAT_VECTOR ? OrigOperand.getOperand(i: 0)
20777 : OrigOperand.getOperand(i: 1);
20778
20779 // See if we have enough sign bits or zero bits in the scalar to use a
20780 // widening opcode by splatting to smaller element size.
20781 unsigned EltBits = VT.getScalarSizeInBits();
20782 unsigned ScalarBits = Op.getValueSizeInBits();
20783 // If we're not getting all bits from the element, we need special handling.
20784 if (ScalarBits < EltBits) {
20785 // This should only occur on RV32.
20786 assert(Opc == RISCVISD::VMV_V_X_VL && EltBits == 64 && ScalarBits == 32 &&
20787 !Subtarget.is64Bit() && "Unexpected splat");
20788 // vmv.v.x sign extends narrow inputs.
20789 SupportsSExt = true;
20790
20791 // If the input is positive, then sign extend is also zero extend.
20792 if (DAG.SignBitIsZero(Op))
20793 SupportsZExt = true;
20794
20795 EnforceOneUse = false;
20796 return;
20797 }
20798
20799 unsigned NarrowSize = EltBits / 2;
20800 // If the narrow type cannot be expressed with a legal VMV,
20801 // this is not a valid candidate.
20802 if (NarrowSize < 8)
20803 return;
20804
20805 if (DAG.ComputeMaxSignificantBits(Op) <= NarrowSize)
20806 SupportsSExt = true;
20807
20808 if (DAG.MaskedValueIsZero(Op,
20809 Mask: APInt::getBitsSetFrom(numBits: ScalarBits, loBit: NarrowSize)))
20810 SupportsZExt = true;
20811
20812 EnforceOneUse = false;
20813 }
20814
20815 bool isSupportedFPExtend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
20816 return (NarrowEltVT == MVT::f32 ||
20817 (NarrowEltVT == MVT::f16 && Subtarget.hasVInstructionsF16()));
20818 }
20819
20820 bool isSupportedBF16Extend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
20821 return NarrowEltVT == MVT::bf16 &&
20822 (Subtarget.hasStdExtZvfbfwma() || Subtarget.hasVInstructionsBF16());
20823 }
20824
20825 /// Helper method to set the various fields of this struct based on the
20826 /// type of \p Root.
20827 void fillUpExtensionSupport(SDNode *Root, SelectionDAG &DAG,
20828 const RISCVSubtarget &Subtarget) {
20829 SupportsZExt = false;
20830 SupportsSExt = false;
20831 SupportsFPExt = false;
20832 SupportsBF16Ext = false;
20833 EnforceOneUse = true;
20834 unsigned Opc = OrigOperand.getOpcode();
20835 // For the nodes we handle below, we end up using their inputs directly: see
20836 // getSource(). However since they either don't have a passthru or we check
20837 // that their passthru is undef, we can safely ignore their mask and VL.
20838 switch (Opc) {
20839 case ISD::ZERO_EXTEND:
20840 case ISD::SIGN_EXTEND: {
20841 MVT VT = OrigOperand.getSimpleValueType();
20842 if (!VT.isVector())
20843 break;
20844
20845 SDValue NarrowElt = OrigOperand.getOperand(i: 0);
20846 MVT NarrowVT = NarrowElt.getSimpleValueType();
20847 // i1 types are legal but we can't select V{S,Z}EXT_VLs with them.
20848 if (NarrowVT.getVectorElementType() == MVT::i1)
20849 break;
20850
20851 SupportsZExt = Opc == ISD::ZERO_EXTEND;
20852 SupportsSExt = Opc == ISD::SIGN_EXTEND;
20853 break;
20854 }
20855 case RISCVISD::VZEXT_VL:
20856 SupportsZExt = true;
20857 break;
20858 case RISCVISD::VSEXT_VL:
20859 SupportsSExt = true;
20860 break;
20861 case RISCVISD::FP_EXTEND_VL: {
20862 MVT NarrowEltVT =
20863 OrigOperand.getOperand(i: 0).getSimpleValueType().getVectorElementType();
20864 if (isSupportedFPExtend(NarrowEltVT, Subtarget))
20865 SupportsFPExt = true;
20866 if (isSupportedBF16Extend(NarrowEltVT, Subtarget))
20867 SupportsBF16Ext = true;
20868
20869 break;
20870 }
20871 case ISD::SPLAT_VECTOR:
20872 case RISCVISD::VMV_V_X_VL:
20873 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
20874 break;
20875 case RISCVISD::VFMV_V_F_VL: {
20876 MVT VT = OrigOperand.getSimpleValueType();
20877
20878 if (!OrigOperand.getOperand(i: 0).isUndef())
20879 break;
20880
20881 SDValue Op = OrigOperand.getOperand(i: 1);
20882 if (Op.getOpcode() != ISD::FP_EXTEND)
20883 break;
20884
20885 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
20886 unsigned ScalarBits = Op.getOperand(i: 0).getValueSizeInBits();
20887 if (NarrowSize != ScalarBits)
20888 break;
20889
20890 if (isSupportedFPExtend(NarrowEltVT: Op.getOperand(i: 0).getSimpleValueType(), Subtarget))
20891 SupportsFPExt = true;
20892 if (isSupportedBF16Extend(NarrowEltVT: Op.getOperand(i: 0).getSimpleValueType(),
20893 Subtarget))
20894 SupportsBF16Ext = true;
20895 break;
20896 }
20897 default:
20898 break;
20899 }
20900 }
20901
20902 /// Check if \p Root supports any extension folding combines.
20903 static bool isSupportedRoot(const SDNode *Root,
20904 const RISCVSubtarget &Subtarget) {
20905 switch (Root->getOpcode()) {
20906 case ISD::ADD:
20907 case ISD::SUB:
20908 case ISD::MUL: {
20909 return Root->getValueType(ResNo: 0).isScalableVector();
20910 }
20911 case ISD::OR: {
20912 return Root->getValueType(ResNo: 0).isScalableVector() &&
20913 Root->getFlags().hasDisjoint();
20914 }
20915 // Vector Widening Integer Add/Sub/Mul Instructions
20916 case RISCVISD::ADD_VL:
20917 case RISCVISD::MUL_VL:
20918 case RISCVISD::VWADD_W_VL:
20919 case RISCVISD::VWADDU_W_VL:
20920 case RISCVISD::SUB_VL:
20921 case RISCVISD::VWSUB_W_VL:
20922 case RISCVISD::VWSUBU_W_VL:
20923 // Vector Widening Floating-Point Add/Sub/Mul Instructions
20924 case RISCVISD::FADD_VL:
20925 case RISCVISD::FSUB_VL:
20926 case RISCVISD::FMUL_VL:
20927 case RISCVISD::VFWADD_W_VL:
20928 case RISCVISD::VFWSUB_W_VL:
20929 return true;
20930 case RISCVISD::OR_VL:
20931 return Root->getFlags().hasDisjoint();
20932 case ISD::SHL:
20933 return Root->getValueType(ResNo: 0).isScalableVector() &&
20934 Subtarget.hasStdExtZvbb();
20935 case RISCVISD::SHL_VL:
20936 return Subtarget.hasStdExtZvbb();
20937 case RISCVISD::VFMADD_VL:
20938 case RISCVISD::VFNMSUB_VL:
20939 case RISCVISD::VFNMADD_VL:
20940 case RISCVISD::VFMSUB_VL:
20941 return true;
20942 default:
20943 return false;
20944 }
20945 }
20946
20947 /// Build a NodeExtensionHelper for \p Root.getOperand(\p OperandIdx).
20948 NodeExtensionHelper(SDNode *Root, unsigned OperandIdx, SelectionDAG &DAG,
20949 const RISCVSubtarget &Subtarget) {
20950 assert(isSupportedRoot(Root, Subtarget) &&
20951 "Trying to build an helper with an "
20952 "unsupported root");
20953 assert(OperandIdx < 2 && "Requesting something else than LHS or RHS");
20954 assert(DAG.getTargetLoweringInfo().isTypeLegal(Root->getValueType(0)));
20955 OrigOperand = Root->getOperand(Num: OperandIdx);
20956
20957 unsigned Opc = Root->getOpcode();
20958 switch (Opc) {
20959 // We consider
20960 // VW<ADD|SUB>_W(LHS, RHS) -> <ADD|SUB>(LHS, SEXT(RHS))
20961 // VW<ADD|SUB>U_W(LHS, RHS) -> <ADD|SUB>(LHS, ZEXT(RHS))
20962 // VFW<ADD|SUB>_W(LHS, RHS) -> F<ADD|SUB>(LHS, FPEXT(RHS))
20963 case RISCVISD::VWADD_W_VL:
20964 case RISCVISD::VWADDU_W_VL:
20965 case RISCVISD::VWSUB_W_VL:
20966 case RISCVISD::VWSUBU_W_VL:
20967 case RISCVISD::VFWADD_W_VL:
20968 case RISCVISD::VFWSUB_W_VL:
20969 // Operand 1 can't be changed.
20970 if (OperandIdx == 1)
20971 break;
20972 [[fallthrough]];
20973 default:
20974 fillUpExtensionSupport(Root, DAG, Subtarget);
20975 break;
20976 }
20977 }
20978
20979 /// Helper function to get the Mask and VL from \p Root.
20980 static std::pair<SDValue, SDValue>
20981 getMaskAndVL(const SDNode *Root, SelectionDAG &DAG,
20982 const RISCVSubtarget &Subtarget) {
20983 assert(isSupportedRoot(Root, Subtarget) && "Unexpected root");
20984 switch (Root->getOpcode()) {
20985 case ISD::ADD:
20986 case ISD::SUB:
20987 case ISD::MUL:
20988 case ISD::OR:
20989 case ISD::SHL: {
20990 SDLoc DL(Root);
20991 MVT VT = Root->getSimpleValueType(ResNo: 0);
20992 return getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
20993 }
20994 default:
20995 return std::make_pair(x: Root->getOperand(Num: 3), y: Root->getOperand(Num: 4));
20996 }
20997 }
20998
20999 /// Helper function to check if \p N is commutative with respect to the
21000 /// foldings that are supported by this class.
21001 static bool isCommutative(const SDNode *N) {
21002 switch (N->getOpcode()) {
21003 case ISD::ADD:
21004 case ISD::MUL:
21005 case ISD::OR:
21006 case RISCVISD::ADD_VL:
21007 case RISCVISD::MUL_VL:
21008 case RISCVISD::OR_VL:
21009 case RISCVISD::FADD_VL:
21010 case RISCVISD::FMUL_VL:
21011 case RISCVISD::VFMADD_VL:
21012 case RISCVISD::VFNMSUB_VL:
21013 case RISCVISD::VFNMADD_VL:
21014 case RISCVISD::VFMSUB_VL:
21015 return true;
21016 case RISCVISD::VWADD_W_VL:
21017 case RISCVISD::VWADDU_W_VL:
21018 case ISD::SUB:
21019 case RISCVISD::SUB_VL:
21020 case RISCVISD::VWSUB_W_VL:
21021 case RISCVISD::VWSUBU_W_VL:
21022 case RISCVISD::VFWADD_W_VL:
21023 case RISCVISD::FSUB_VL:
21024 case RISCVISD::VFWSUB_W_VL:
21025 case ISD::SHL:
21026 case RISCVISD::SHL_VL:
21027 return false;
21028 default:
21029 llvm_unreachable("Unexpected opcode");
21030 }
21031 }
21032
21033 /// Get a list of combine to try for folding extensions in \p Root.
21034 /// Note that each returned CombineToTry function doesn't actually modify
21035 /// anything. Instead they produce an optional CombineResult that if not None,
21036 /// need to be materialized for the combine to be applied.
21037 /// \see CombineResult::materialize.
21038 /// If the related CombineToTry function returns std::nullopt, that means the
21039 /// combine didn't match.
21040 static SmallVector<CombineToTry>
21041 getSupportedFoldings(const SDNode *Root, const RISCVSubtarget &Subtarget);
21042};
21043
21044/// Helper structure that holds all the necessary information to materialize a
21045/// combine that does some extension folding.
21046struct CombineResult {
21047 /// Opcode to be generated when materializing the combine.
21048 unsigned TargetOpcode;
21049 // No value means no extension is needed.
21050 std::optional<ExtKind> LHSExt;
21051 std::optional<ExtKind> RHSExt;
21052 /// Root of the combine.
21053 SDNode *Root;
21054 /// LHS of the TargetOpcode.
21055 NodeExtensionHelper LHS;
21056 /// RHS of the TargetOpcode.
21057 NodeExtensionHelper RHS;
21058
21059 CombineResult(unsigned TargetOpcode, SDNode *Root,
21060 const NodeExtensionHelper &LHS, std::optional<ExtKind> LHSExt,
21061 const NodeExtensionHelper &RHS, std::optional<ExtKind> RHSExt)
21062 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
21063 LHS(LHS), RHS(RHS) {}
21064
21065 /// Return a value that uses TargetOpcode and that can be used to replace
21066 /// Root.
21067 /// The actual replacement is *not* done in that method.
21068 SDValue materialize(SelectionDAG &DAG,
21069 const RISCVSubtarget &Subtarget) const {
21070 SDValue Mask, VL, Passthru;
21071 std::tie(args&: Mask, args&: VL) =
21072 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
21073 switch (Root->getOpcode()) {
21074 default:
21075 Passthru = Root->getOperand(Num: 2);
21076 break;
21077 case ISD::ADD:
21078 case ISD::SUB:
21079 case ISD::MUL:
21080 case ISD::OR:
21081 case ISD::SHL:
21082 Passthru = DAG.getUNDEF(VT: Root->getValueType(ResNo: 0));
21083 break;
21084 }
21085 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(Root), VT: Root->getValueType(ResNo: 0),
21086 N1: LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SupportsExt: LHSExt),
21087 N2: RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SupportsExt: RHSExt),
21088 N3: Passthru, N4: Mask, N5: VL);
21089 }
21090};
21091
21092/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
21093/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
21094/// are zext) and LHS and RHS can be folded into Root.
21095/// AllowExtMask define which form `ext` can take in this pattern.
21096///
21097/// \note If the pattern can match with both zext and sext, the returned
21098/// CombineResult will feature the zext result.
21099///
21100/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21101/// can be used to apply the pattern.
21102static std::optional<CombineResult>
21103canFoldToVWWithSameExtensionImpl(SDNode *Root, const NodeExtensionHelper &LHS,
21104 const NodeExtensionHelper &RHS,
21105 uint8_t AllowExtMask, SelectionDAG &DAG,
21106 const RISCVSubtarget &Subtarget) {
21107 if ((AllowExtMask & ExtKind::ZExt) && LHS.SupportsZExt && RHS.SupportsZExt)
21108 return CombineResult(NodeExtensionHelper::getZExtOpcode(Opcode: Root->getOpcode()),
21109 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
21110 /*RHSExt=*/{ExtKind::ZExt});
21111 if ((AllowExtMask & ExtKind::SExt) && LHS.SupportsSExt && RHS.SupportsSExt)
21112 return CombineResult(NodeExtensionHelper::getSExtOpcode(Opcode: Root->getOpcode()),
21113 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
21114 /*RHSExt=*/{ExtKind::SExt});
21115 if ((AllowExtMask & ExtKind::FPExt) && LHS.SupportsFPExt && RHS.SupportsFPExt)
21116 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
21117 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
21118 /*RHSExt=*/{ExtKind::FPExt});
21119 if ((AllowExtMask & ExtKind::BF16Ext) && LHS.SupportsBF16Ext &&
21120 RHS.SupportsBF16Ext)
21121 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
21122 Root, LHS, /*LHSExt=*/{ExtKind::BF16Ext}, RHS,
21123 /*RHSExt=*/{ExtKind::BF16Ext});
21124 return std::nullopt;
21125}
21126
21127/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
21128/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
21129/// are zext) and LHS and RHS can be folded into Root.
21130///
21131/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21132/// can be used to apply the pattern.
21133static std::optional<CombineResult>
21134canFoldToVWWithSameExtension(SDNode *Root, const NodeExtensionHelper &LHS,
21135 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21136 const RISCVSubtarget &Subtarget) {
21137 return canFoldToVWWithSameExtensionImpl(
21138 Root, LHS, RHS, AllowExtMask: ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
21139 Subtarget);
21140}
21141
21142/// Check if \p Root follows a pattern Root(zext(LHS), zext(RHS))
21143///
21144/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21145/// can be used to apply the pattern.
21146static std::optional<CombineResult>
21147canFoldToVWWithSameExtZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
21148 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21149 const RISCVSubtarget &Subtarget) {
21150 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, AllowExtMask: ExtKind::ZExt, DAG,
21151 Subtarget);
21152}
21153
21154/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS))
21155///
21156/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21157/// can be used to apply the pattern.
21158static std::optional<CombineResult>
21159canFoldToVWWithSameExtBF16(SDNode *Root, const NodeExtensionHelper &LHS,
21160 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21161 const RISCVSubtarget &Subtarget) {
21162 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, AllowExtMask: ExtKind::BF16Ext, DAG,
21163 Subtarget);
21164}
21165
21166/// Check if \p Root follows a pattern Root(LHS, ext(RHS))
21167///
21168/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21169/// can be used to apply the pattern.
21170static std::optional<CombineResult>
21171canFoldToVW_W(SDNode *Root, const NodeExtensionHelper &LHS,
21172 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21173 const RISCVSubtarget &Subtarget) {
21174 if (RHS.SupportsFPExt)
21175 return CombineResult(
21176 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::FPExt),
21177 Root, LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::FPExt});
21178
21179 // FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
21180 // sext/zext?
21181 // Control this behavior behind an option (AllowSplatInVW_W) for testing
21182 // purposes.
21183 if (RHS.SupportsZExt && (!RHS.isSplat() || AllowSplatInVW_W))
21184 return CombineResult(
21185 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::ZExt), Root,
21186 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::ZExt});
21187 if (RHS.SupportsSExt && (!RHS.isSplat() || AllowSplatInVW_W))
21188 return CombineResult(
21189 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::SExt), Root,
21190 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::SExt});
21191 return std::nullopt;
21192}
21193
21194/// Check if \p Root follows a pattern Root(sext(LHS), RHS)
21195///
21196/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21197/// can be used to apply the pattern.
21198static std::optional<CombineResult>
21199canFoldToVWWithSEXT(SDNode *Root, const NodeExtensionHelper &LHS,
21200 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21201 const RISCVSubtarget &Subtarget) {
21202 if (LHS.SupportsSExt)
21203 return CombineResult(NodeExtensionHelper::getSExtOpcode(Opcode: Root->getOpcode()),
21204 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
21205 /*RHSExt=*/std::nullopt);
21206 return std::nullopt;
21207}
21208
21209/// Check if \p Root follows a pattern Root(zext(LHS), RHS)
21210///
21211/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21212/// can be used to apply the pattern.
21213static std::optional<CombineResult>
21214canFoldToVWWithZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
21215 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21216 const RISCVSubtarget &Subtarget) {
21217 if (LHS.SupportsZExt)
21218 return CombineResult(NodeExtensionHelper::getZExtOpcode(Opcode: Root->getOpcode()),
21219 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
21220 /*RHSExt=*/std::nullopt);
21221 return std::nullopt;
21222}
21223
21224/// Check if \p Root follows a pattern Root(fpext(LHS), RHS)
21225///
21226/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21227/// can be used to apply the pattern.
21228static std::optional<CombineResult>
21229canFoldToVWWithFPEXT(SDNode *Root, const NodeExtensionHelper &LHS,
21230 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21231 const RISCVSubtarget &Subtarget) {
21232 if (LHS.SupportsFPExt)
21233 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
21234 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
21235 /*RHSExt=*/std::nullopt);
21236 return std::nullopt;
21237}
21238
21239/// Check if \p Root follows a pattern Root(sext(LHS), zext(RHS))
21240///
21241/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
21242/// can be used to apply the pattern.
21243static std::optional<CombineResult>
21244canFoldToVW_SU(SDNode *Root, const NodeExtensionHelper &LHS,
21245 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
21246 const RISCVSubtarget &Subtarget) {
21247
21248 if (!LHS.SupportsSExt || !RHS.SupportsZExt)
21249 return std::nullopt;
21250 return CombineResult(NodeExtensionHelper::getSUOpcode(Opcode: Root->getOpcode()),
21251 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
21252 /*RHSExt=*/{ExtKind::ZExt});
21253}
21254
21255SmallVector<NodeExtensionHelper::CombineToTry>
21256NodeExtensionHelper::getSupportedFoldings(const SDNode *Root,
21257 const RISCVSubtarget &Subtarget) {
21258 SmallVector<CombineToTry> Strategies;
21259 switch (Root->getOpcode()) {
21260 case ISD::ADD:
21261 case ISD::SUB:
21262 case ISD::OR:
21263 case RISCVISD::ADD_VL:
21264 case RISCVISD::SUB_VL:
21265 case RISCVISD::OR_VL:
21266 case RISCVISD::FADD_VL:
21267 case RISCVISD::FSUB_VL:
21268 // add|sub|fadd|fsub-> vwadd(u)|vwsub(u)|vfwadd|vfwsub
21269 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
21270 if (Subtarget.hasVInstructionsBF16())
21271 Strategies.push_back(Elt: canFoldToVWWithSameExtBF16);
21272 // add|sub|fadd|fsub -> vwadd(u)_w|vwsub(u)_w}|vfwadd_w|vfwsub_w
21273 Strategies.push_back(Elt: canFoldToVW_W);
21274 break;
21275 case RISCVISD::FMUL_VL:
21276 case RISCVISD::VFMADD_VL:
21277 case RISCVISD::VFMSUB_VL:
21278 case RISCVISD::VFNMADD_VL:
21279 case RISCVISD::VFNMSUB_VL:
21280 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
21281 if (Subtarget.hasVInstructionsBF16() ||
21282 (Subtarget.hasStdExtZvfbfwma() &&
21283 Root->getOpcode() == RISCVISD::VFMADD_VL))
21284 Strategies.push_back(Elt: canFoldToVWWithSameExtBF16);
21285 break;
21286 case ISD::MUL:
21287 case RISCVISD::MUL_VL:
21288 // mul -> vwmul(u)
21289 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
21290 // mul -> vwmulsu
21291 Strategies.push_back(Elt: canFoldToVW_SU);
21292 break;
21293 case ISD::SHL:
21294 case RISCVISD::SHL_VL:
21295 // shl -> vwsll
21296 Strategies.push_back(Elt: canFoldToVWWithSameExtZEXT);
21297 break;
21298 case RISCVISD::VWADD_W_VL:
21299 case RISCVISD::VWSUB_W_VL:
21300 // vwadd_w|vwsub_w -> vwadd|vwsub
21301 Strategies.push_back(Elt: canFoldToVWWithSEXT);
21302 break;
21303 case RISCVISD::VWADDU_W_VL:
21304 case RISCVISD::VWSUBU_W_VL:
21305 // vwaddu_w|vwsubu_w -> vwaddu|vwsubu
21306 Strategies.push_back(Elt: canFoldToVWWithZEXT);
21307 break;
21308 case RISCVISD::VFWADD_W_VL:
21309 case RISCVISD::VFWSUB_W_VL:
21310 // vfwadd_w|vfwsub_w -> vfwadd|vfwsub
21311 Strategies.push_back(Elt: canFoldToVWWithFPEXT);
21312 break;
21313 default:
21314 llvm_unreachable("Unexpected opcode");
21315 }
21316 return Strategies;
21317}
21318} // End anonymous namespace.
21319
21320static SDValue simplifyOp_VL(SDNode *N) {
21321 // TODO: Extend this to other binops using generic identity logic
21322 assert(N->getOpcode() == RISCVISD::ADD_VL);
21323 SDValue A = N->getOperand(Num: 0);
21324 SDValue B = N->getOperand(Num: 1);
21325 SDValue Passthru = N->getOperand(Num: 2);
21326 if (!Passthru.isUndef())
21327 // TODO:This could be a vmerge instead
21328 return SDValue();
21329 ;
21330 if (ISD::isConstantSplatVectorAllZeros(N: B.getNode()))
21331 return A;
21332 // Peek through fixed to scalable
21333 if (B.getOpcode() == ISD::INSERT_SUBVECTOR && B.getOperand(i: 0).isUndef() &&
21334 ISD::isConstantSplatVectorAllZeros(N: B.getOperand(i: 1).getNode()))
21335 return A;
21336 return SDValue();
21337}
21338
21339/// Combine a binary or FMA operation to its equivalent VW or VW_W form.
21340/// The supported combines are:
21341/// add | add_vl | or disjoint | or_vl disjoint -> vwadd(u) | vwadd(u)_w
21342/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
21343/// mul | mul_vl -> vwmul(u) | vwmul_su
21344/// shl | shl_vl -> vwsll
21345/// fadd_vl -> vfwadd | vfwadd_w
21346/// fsub_vl -> vfwsub | vfwsub_w
21347/// fmul_vl -> vfwmul
21348/// vwadd_w(u) -> vwadd(u)
21349/// vwsub_w(u) -> vwsub(u)
21350/// vfwadd_w -> vfwadd
21351/// vfwsub_w -> vfwsub
21352static SDValue combineOp_VLToVWOp_VL(SDNode *N,
21353 TargetLowering::DAGCombinerInfo &DCI,
21354 const RISCVSubtarget &Subtarget) {
21355 SelectionDAG &DAG = DCI.DAG;
21356 if (DCI.isBeforeLegalize())
21357 return SDValue();
21358
21359 if (!NodeExtensionHelper::isSupportedRoot(Root: N, Subtarget))
21360 return SDValue();
21361
21362 SmallVector<SDNode *> Worklist;
21363 SmallPtrSet<SDNode *, 8> Inserted;
21364 SmallPtrSet<SDNode *, 8> ExtensionsToRemove;
21365 Worklist.push_back(Elt: N);
21366 Inserted.insert(Ptr: N);
21367 SmallVector<CombineResult> CombinesToApply;
21368
21369 while (!Worklist.empty()) {
21370 SDNode *Root = Worklist.pop_back_val();
21371
21372 NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
21373 NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
21374 auto AppendUsersIfNeeded =
21375 [&Worklist, &Subtarget, &Inserted,
21376 &ExtensionsToRemove](const NodeExtensionHelper &Op) {
21377 if (Op.needToPromoteOtherUsers()) {
21378 // Remember that we're supposed to remove this extension.
21379 ExtensionsToRemove.insert(Ptr: Op.OrigOperand.getNode());
21380 for (SDUse &Use : Op.OrigOperand->uses()) {
21381 SDNode *TheUser = Use.getUser();
21382 if (!NodeExtensionHelper::isSupportedRoot(Root: TheUser, Subtarget))
21383 return false;
21384 // We only support the first 2 operands of FMA.
21385 if (Use.getOperandNo() >= 2)
21386 return false;
21387 if (Inserted.insert(Ptr: TheUser).second)
21388 Worklist.push_back(Elt: TheUser);
21389 }
21390 }
21391 return true;
21392 };
21393
21394 // Control the compile time by limiting the number of node we look at in
21395 // total.
21396 if (Inserted.size() > ExtensionMaxWebSize)
21397 return SDValue();
21398
21399 SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
21400 NodeExtensionHelper::getSupportedFoldings(Root, Subtarget);
21401
21402 assert(!FoldingStrategies.empty() && "Nothing to be folded");
21403 bool Matched = false;
21404 for (int Attempt = 0;
21405 (Attempt != 1 + NodeExtensionHelper::isCommutative(N: Root)) && !Matched;
21406 ++Attempt) {
21407
21408 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
21409 FoldingStrategies) {
21410 std::optional<CombineResult> Res =
21411 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
21412 if (Res) {
21413 // If this strategy wouldn't remove an extension we're supposed to
21414 // remove, reject it.
21415 if (!Res->LHSExt.has_value() &&
21416 ExtensionsToRemove.contains(Ptr: LHS.OrigOperand.getNode()))
21417 continue;
21418 if (!Res->RHSExt.has_value() &&
21419 ExtensionsToRemove.contains(Ptr: RHS.OrigOperand.getNode()))
21420 continue;
21421
21422 Matched = true;
21423 CombinesToApply.push_back(Elt: *Res);
21424 // All the inputs that are extended need to be folded, otherwise
21425 // we would be leaving the old input (since it is may still be used),
21426 // and the new one.
21427 if (Res->LHSExt.has_value())
21428 if (!AppendUsersIfNeeded(LHS))
21429 return SDValue();
21430 if (Res->RHSExt.has_value())
21431 if (!AppendUsersIfNeeded(RHS))
21432 return SDValue();
21433 break;
21434 }
21435 }
21436 std::swap(a&: LHS, b&: RHS);
21437 }
21438 // Right now we do an all or nothing approach.
21439 if (!Matched)
21440 return SDValue();
21441 }
21442 // Store the value for the replacement of the input node separately.
21443 SDValue InputRootReplacement;
21444 // We do the RAUW after we materialize all the combines, because some replaced
21445 // nodes may be feeding some of the yet-to-be-replaced nodes. Put differently,
21446 // some of these nodes may appear in the NodeExtensionHelpers of some of the
21447 // yet-to-be-visited CombinesToApply roots.
21448 SmallVector<std::pair<SDValue, SDValue>> ValuesToReplace;
21449 ValuesToReplace.reserve(N: CombinesToApply.size());
21450 for (CombineResult Res : CombinesToApply) {
21451 SDValue NewValue = Res.materialize(DAG, Subtarget);
21452 if (!InputRootReplacement) {
21453 assert(Res.Root == N &&
21454 "First element is expected to be the current node");
21455 InputRootReplacement = NewValue;
21456 } else {
21457 ValuesToReplace.emplace_back(Args: SDValue(Res.Root, 0), Args&: NewValue);
21458 }
21459 }
21460 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
21461 DCI.CombineTo(N: OldNewValues.first.getNode(), Res: OldNewValues.second);
21462 }
21463 return InputRootReplacement;
21464}
21465
21466// Fold (vwadd(u).wv y, (vmerge cond, x, 0)) -> vwadd(u).wv y, x, y, cond
21467// (vwsub(u).wv y, (vmerge cond, x, 0)) -> vwsub(u).wv y, x, y, cond
21468// y will be the Passthru and cond will be the Mask.
21469static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG) {
21470 unsigned Opc = N->getOpcode();
21471 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
21472 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
21473
21474 SDValue Y = N->getOperand(Num: 0);
21475 SDValue MergeOp = N->getOperand(Num: 1);
21476 unsigned MergeOpc = MergeOp.getOpcode();
21477
21478 if (MergeOpc != RISCVISD::VMERGE_VL && MergeOpc != ISD::VSELECT)
21479 return SDValue();
21480
21481 SDValue X = MergeOp->getOperand(Num: 1);
21482
21483 if (!MergeOp.hasOneUse())
21484 return SDValue();
21485
21486 // Passthru should be undef
21487 SDValue Passthru = N->getOperand(Num: 2);
21488 if (!Passthru.isUndef())
21489 return SDValue();
21490
21491 // Mask should be all ones
21492 SDValue Mask = N->getOperand(Num: 3);
21493 if (Mask.getOpcode() != RISCVISD::VMSET_VL)
21494 return SDValue();
21495
21496 // False value of MergeOp should be all zeros
21497 SDValue Z = MergeOp->getOperand(Num: 2);
21498
21499 if (Z.getOpcode() == ISD::INSERT_SUBVECTOR &&
21500 (isNullOrNullSplat(V: Z.getOperand(i: 0)) || Z.getOperand(i: 0).isUndef()))
21501 Z = Z.getOperand(i: 1);
21502
21503 if (!ISD::isConstantSplatVectorAllZeros(N: Z.getNode()))
21504 return SDValue();
21505
21506 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
21507 Ops: {Y, X, Y, MergeOp->getOperand(Num: 0), N->getOperand(Num: 4)},
21508 Flags: N->getFlags());
21509}
21510
21511// vwaddu C (vabd A B) -> vwabda(A B C)
21512// vwaddu C (vabdu A B) -> vwabdau(A B C)
21513static SDValue performVWABDACombine(SDNode *N, SelectionDAG &DAG,
21514 const RISCVSubtarget &Subtarget) {
21515 if (!Subtarget.hasStdExtZvabd())
21516 return SDValue();
21517
21518 MVT VT = N->getSimpleValueType(ResNo: 0);
21519 if (VT.getVectorElementType() != MVT::i16 &&
21520 VT.getVectorElementType() != MVT::i32)
21521 return SDValue();
21522
21523 SDValue Op0 = N->getOperand(Num: 0);
21524 SDValue Op1 = N->getOperand(Num: 1);
21525 SDValue Passthru = N->getOperand(Num: 2);
21526 if (!Passthru->isUndef())
21527 return SDValue();
21528
21529 SDValue Mask = N->getOperand(Num: 3);
21530 SDValue VL = N->getOperand(Num: 4);
21531 auto IsCombinableABD = [&](SDValue Op) {
21532 if (Op->getOpcode() != RISCVISD::ABDS_VL &&
21533 Op->getOpcode() != RISCVISD::ABDU_VL)
21534 return SDValue();
21535 // The combine is only valid when both instructions share the same mask and
21536 // VL.
21537 if (!Op.getOperand(i: 2).isUndef() || Op.getOperand(i: 3) != Mask ||
21538 Op.getOperand(i: 4) != VL)
21539 return SDValue();
21540 return Op;
21541 };
21542
21543 SDValue Diff = IsCombinableABD(Op0);
21544 Diff = Diff ? Diff : IsCombinableABD(Op1);
21545 if (!Diff)
21546 return SDValue();
21547 SDValue Acc = Diff == Op0 ? Op1 : Op0;
21548
21549 SDLoc DL(N);
21550 Acc = DAG.getNode(Opcode: RISCVISD::VZEXT_VL, DL, VT, N1: Acc, N2: Mask, N3: VL);
21551 SDValue Result = DAG.getNode(
21552 Opcode: Diff.getOpcode() == RISCVISD::ABDS_VL ? RISCVISD::VWABDA_VL
21553 : RISCVISD::VWABDAU_VL,
21554 DL, VT, N1: Diff.getOperand(i: 0), N2: Diff.getOperand(i: 1), N3: Acc, N4: Mask, N5: VL);
21555 return Result;
21556}
21557
21558// vwaddu_wv C (vabd A B) -> vwabda(A B C)
21559// vwaddu_wv C (zext (vabd A B)) -> vwabda(A (sext B) (sext C))
21560// vwaddu_wv C (vabdu A B) -> vwabdau(A B C)
21561// vwaddu_wv C (zext (vabdu A B)) -> vwabdau(A (zext B) (zext C))
21562static SDValue performVWABDACombineWV(SDNode *N, SelectionDAG &DAG,
21563 const RISCVSubtarget &Subtarget) {
21564 if (!Subtarget.hasStdExtZvabd())
21565 return SDValue();
21566
21567 MVT VT = N->getSimpleValueType(ResNo: 0);
21568 // The result is widened, so we can accept i16/i32 here.
21569 if (VT.getVectorElementType() != MVT::i16 &&
21570 VT.getVectorElementType() != MVT::i32)
21571 return SDValue();
21572
21573 SDValue Acc = N->getOperand(Num: 0);
21574 SDValue Op1 = N->getOperand(Num: 1);
21575 SDValue Passthru = N->getOperand(Num: 2);
21576 if (!Passthru->isUndef())
21577 return SDValue();
21578
21579 SDValue Mask = N->getOperand(Num: 3);
21580 SDValue VL = N->getOperand(Num: 4);
21581 unsigned ExtOpc = 0;
21582 MVT ExtVT;
21583 auto IsCombinableABD = [&](SDValue Op) {
21584 unsigned Opc = Op.getOpcode();
21585 if (Opc == ISD::ABDS || Opc == ISD::ABDU)
21586 return true;
21587 if (Opc != RISCVISD::ABDS_VL && Opc != RISCVISD::ABDU_VL)
21588 return false;
21589 return Op.getOperand(i: 2).isUndef() && Op.getOperand(i: 3) == Mask &&
21590 Op.getOperand(i: 4) == VL;
21591 };
21592 auto GetDiff = [&](SDValue Op) {
21593 unsigned Opc = Op.getOpcode();
21594 if (Opc == RISCVISD::VZEXT_VL) {
21595 if (Op.getOperand(i: 1) != Mask || Op.getOperand(i: 2) != VL)
21596 return SDValue();
21597 SDValue Src = Op->getOperand(Num: 0);
21598 unsigned SrcOpc = Src.getOpcode();
21599 switch (SrcOpc) {
21600 default:
21601 return SDValue();
21602 case ISD::ABDS:
21603 case RISCVISD::ABDS_VL:
21604 ExtOpc = RISCVISD::VSEXT_VL;
21605 break;
21606 case ISD::ABDU:
21607 case RISCVISD::ABDU_VL:
21608 ExtOpc = RISCVISD::VZEXT_VL;
21609 break;
21610 }
21611 if (!IsCombinableABD(Src))
21612 return SDValue();
21613 ExtVT = Op->getSimpleValueType(ResNo: 0);
21614 return Src;
21615 }
21616
21617 if (!IsCombinableABD(Op))
21618 return SDValue();
21619 return Op;
21620 };
21621
21622 // VWADDU_W_VL is not commutative, operand 0 is wide and operand 1 is
21623 // narrow. Only the narrow operand can be folded into vwabda/vwabdau.
21624 SDValue Diff = GetDiff(Op1);
21625 if (!Diff)
21626 return SDValue();
21627
21628 SDLoc DL(N);
21629 SDValue DiffA = Diff.getOperand(i: 0);
21630 SDValue DiffB = Diff.getOperand(i: 1);
21631 if (ExtOpc) {
21632 DiffA = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtVT, N1: DiffA, N2: Mask, N3: VL);
21633 DiffB = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtVT, N1: DiffB, N2: Mask, N3: VL);
21634 }
21635 SDValue Result = DAG.getNode(Opcode: Diff.getOpcode() == ISD::ABDS ||
21636 Diff.getOpcode() == RISCVISD::ABDS_VL
21637 ? RISCVISD::VWABDA_VL
21638 : RISCVISD::VWABDAU_VL,
21639 DL, VT, N1: DiffA, N2: DiffB, N3: Acc, N4: Mask, N5: VL);
21640 return Result;
21641}
21642
21643static SDValue performVWADDSUBW_VLCombine(SDNode *N,
21644 TargetLowering::DAGCombinerInfo &DCI,
21645 const RISCVSubtarget &Subtarget) {
21646 [[maybe_unused]] unsigned Opc = N->getOpcode();
21647 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
21648 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
21649
21650 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
21651 return V;
21652
21653 return combineVWADDSUBWSelect(N, DAG&: DCI.DAG);
21654}
21655
21656// Helper function for performMemPairCombine.
21657// Try to combine the memory loads/stores LSNode1 and LSNode2
21658// into a single memory pair operation.
21659static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1,
21660 LSBaseSDNode *LSNode2, SDValue BasePtr,
21661 uint64_t Imm) {
21662 SmallPtrSet<const SDNode *, 32> Visited;
21663 SmallVector<const SDNode *, 8> Worklist = {LSNode1, LSNode2};
21664
21665 if (SDNode::hasPredecessorHelper(N: LSNode1, Visited, Worklist) ||
21666 SDNode::hasPredecessorHelper(N: LSNode2, Visited, Worklist))
21667 return SDValue();
21668
21669 MachineFunction &MF = DAG.getMachineFunction();
21670 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
21671
21672 // The new operation has twice the width.
21673 MVT XLenVT = Subtarget.getXLenVT();
21674 EVT MemVT = LSNode1->getMemoryVT();
21675 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
21676 MachineMemOperand *MMO = LSNode1->getMemOperand();
21677 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
21678 MMO, PtrInfo: MMO->getPointerInfo(), Size: MemVT == MVT::i32 ? 8 : 16);
21679
21680 if (LSNode1->getOpcode() == ISD::LOAD) {
21681 auto Ext = cast<LoadSDNode>(Val: LSNode1)->getExtensionType();
21682 unsigned Opcode;
21683 if (MemVT == MVT::i32)
21684 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD;
21685 else
21686 Opcode = RISCVISD::TH_LDD;
21687
21688 SDValue Res = DAG.getMemIntrinsicNode(
21689 Opcode, dl: SDLoc(LSNode1), VTList: DAG.getVTList(VTs: {XLenVT, XLenVT, MVT::Other}),
21690 Ops: {LSNode1->getChain(), BasePtr,
21691 DAG.getConstant(Val: Imm, DL: SDLoc(LSNode1), VT: XLenVT)},
21692 MemVT: NewMemVT, MMO: NewMMO);
21693
21694 SDValue Node1 =
21695 DAG.getMergeValues(Ops: {Res.getValue(R: 0), Res.getValue(R: 2)}, dl: SDLoc(LSNode1));
21696 SDValue Node2 =
21697 DAG.getMergeValues(Ops: {Res.getValue(R: 1), Res.getValue(R: 2)}, dl: SDLoc(LSNode2));
21698
21699 DAG.ReplaceAllUsesWith(From: LSNode2, To: Node2.getNode());
21700 return Node1;
21701 } else {
21702 unsigned Opcode = (MemVT == MVT::i32) ? RISCVISD::TH_SWD : RISCVISD::TH_SDD;
21703
21704 SDValue Res = DAG.getMemIntrinsicNode(
21705 Opcode, dl: SDLoc(LSNode1), VTList: DAG.getVTList(VT: MVT::Other),
21706 Ops: {LSNode1->getChain(), LSNode1->getOperand(Num: 1), LSNode2->getOperand(Num: 1),
21707 BasePtr, DAG.getConstant(Val: Imm, DL: SDLoc(LSNode1), VT: XLenVT)},
21708 MemVT: NewMemVT, MMO: NewMMO);
21709
21710 DAG.ReplaceAllUsesWith(From: LSNode2, To: Res.getNode());
21711 return Res;
21712 }
21713}
21714
21715// Try to combine two adjacent loads/stores to a single pair instruction from
21716// the XTHeadMemPair vendor extension.
21717static SDValue performMemPairCombine(SDNode *N,
21718 TargetLowering::DAGCombinerInfo &DCI) {
21719 SelectionDAG &DAG = DCI.DAG;
21720 MachineFunction &MF = DAG.getMachineFunction();
21721 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
21722
21723 // Target does not support load/store pair.
21724 if (!Subtarget.hasVendorXTHeadMemPair())
21725 return SDValue();
21726
21727 LSBaseSDNode *LSNode1 = cast<LSBaseSDNode>(Val: N);
21728 EVT MemVT = LSNode1->getMemoryVT();
21729 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2;
21730
21731 // No volatile, indexed or atomic loads/stores.
21732 if (!LSNode1->isSimple() || LSNode1->isIndexed())
21733 return SDValue();
21734
21735 // Function to get a base + constant representation from a memory value.
21736 auto ExtractBaseAndOffset = [](SDValue Ptr) -> std::pair<SDValue, uint64_t> {
21737 if (Ptr->getOpcode() == ISD::ADD)
21738 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1)))
21739 return {Ptr->getOperand(Num: 0), C1->getZExtValue()};
21740 return {Ptr, 0};
21741 };
21742
21743 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->getOperand(Num: OpNum));
21744
21745 SDValue Chain = N->getOperand(Num: 0);
21746 for (SDUse &Use : Chain->uses()) {
21747 if (Use.getUser() != N && Use.getResNo() == 0 &&
21748 Use.getUser()->getOpcode() == N->getOpcode()) {
21749 LSBaseSDNode *LSNode2 = cast<LSBaseSDNode>(Val: Use.getUser());
21750
21751 // No volatile, indexed or atomic loads/stores.
21752 if (!LSNode2->isSimple() || LSNode2->isIndexed())
21753 continue;
21754
21755 // Check if LSNode1 and LSNode2 have the same type and extension.
21756 if (LSNode1->getOpcode() == ISD::LOAD)
21757 if (cast<LoadSDNode>(Val: LSNode2)->getExtensionType() !=
21758 cast<LoadSDNode>(Val: LSNode1)->getExtensionType())
21759 continue;
21760
21761 if (LSNode1->getMemoryVT() != LSNode2->getMemoryVT())
21762 continue;
21763
21764 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->getOperand(Num: OpNum));
21765
21766 // Check if the base pointer is the same for both instruction.
21767 if (Base1 != Base2)
21768 continue;
21769
21770 // Check if the offsets match the XTHeadMemPair encoding constraints.
21771 bool Valid = false;
21772 if (MemVT == MVT::i32) {
21773 // Check for adjacent i32 values and a 2-bit index.
21774 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(x: Offset1))
21775 Valid = true;
21776 } else if (MemVT == MVT::i64) {
21777 // Check for adjacent i64 values and a 2-bit index.
21778 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(x: Offset1))
21779 Valid = true;
21780 }
21781
21782 if (!Valid)
21783 continue;
21784
21785 // Try to combine.
21786 if (SDValue Res =
21787 tryMemPairCombine(DAG, LSNode1, LSNode2, BasePtr: Base1, Imm: Offset1))
21788 return Res;
21789 }
21790 }
21791
21792 return SDValue();
21793}
21794
21795// Fold
21796// (fp_to_int (froundeven X)) -> fcvt X, rne
21797// (fp_to_int (ftrunc X)) -> fcvt X, rtz
21798// (fp_to_int (ffloor X)) -> fcvt X, rdn
21799// (fp_to_int (fceil X)) -> fcvt X, rup
21800// (fp_to_int (fround X)) -> fcvt X, rmm
21801// (fp_to_int (frint X)) -> fcvt X
21802static SDValue performFP_TO_INTCombine(SDNode *N,
21803 TargetLowering::DAGCombinerInfo &DCI,
21804 const RISCVSubtarget &Subtarget) {
21805 SelectionDAG &DAG = DCI.DAG;
21806 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
21807 MVT XLenVT = Subtarget.getXLenVT();
21808
21809 SDValue Src = N->getOperand(Num: 0);
21810
21811 // Don't do this for strict-fp Src.
21812 if (Src->isStrictFPOpcode())
21813 return SDValue();
21814
21815 // Ensure the FP type is legal.
21816 if (!TLI.isTypeLegal(VT: Src.getValueType()))
21817 return SDValue();
21818
21819 // Don't do this for f16 with Zfhmin and not Zfh.
21820 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
21821 return SDValue();
21822
21823 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Src.getOpcode());
21824 // If the result is invalid, we didn't find a foldable instruction.
21825 if (FRM == RISCVFPRndMode::Invalid)
21826 return SDValue();
21827
21828 SDLoc DL(N);
21829 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
21830 EVT VT = N->getValueType(ResNo: 0);
21831
21832 if (VT.isVector() && TLI.isTypeLegal(VT)) {
21833 MVT SrcVT = Src.getSimpleValueType();
21834 MVT SrcContainerVT = SrcVT;
21835 MVT ContainerVT = VT.getSimpleVT();
21836 SDValue XVal = Src.getOperand(i: 0);
21837
21838 // For widening and narrowing conversions we just combine it into a
21839 // VFCVT_..._VL node, as there are no specific VFWCVT/VFNCVT VL nodes. They
21840 // end up getting lowered to their appropriate pseudo instructions based on
21841 // their operand types
21842 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits() * 2 ||
21843 VT.getScalarSizeInBits() * 2 < SrcVT.getScalarSizeInBits())
21844 return SDValue();
21845
21846 // If we'll need to write FRM, don't fold unless the src has a single use.
21847 // This avoids potentially writing FRM multiple times. RTZ has a dedicated
21848 // instruction and DYN uses the current FRM, so neither needs a write.
21849 if (FRM != RISCVFPRndMode::RTZ && FRM != RISCVFPRndMode::DYN &&
21850 !Src.hasOneUse())
21851 return SDValue();
21852
21853 // Make fixed-length vectors scalable first
21854 if (SrcVT.isFixedLengthVector()) {
21855 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
21856 XVal = convertToScalableVector(VT: SrcContainerVT, V: XVal, DAG, Subtarget);
21857 ContainerVT = getContainerForFixedLengthVector(VT: ContainerVT, Subtarget);
21858 }
21859
21860 auto [Mask, VL] =
21861 getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget);
21862
21863 SDValue FpToInt;
21864 if (FRM == RISCVFPRndMode::RTZ) {
21865 // Use the dedicated trunc static rounding mode if we're truncating so we
21866 // don't need to generate calls to fsrmi/fsrm
21867 unsigned Opc =
21868 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
21869 FpToInt = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: XVal, N2: Mask, N3: VL);
21870 } else {
21871 unsigned Opc =
21872 IsSigned ? RISCVISD::VFCVT_RM_X_F_VL : RISCVISD::VFCVT_RM_XU_F_VL;
21873 FpToInt = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: XVal, N2: Mask,
21874 N3: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), N4: VL);
21875 }
21876
21877 // If converted from fixed-length to scalable, convert back
21878 if (VT.isFixedLengthVector())
21879 FpToInt = convertFromScalableVector(VT, V: FpToInt, DAG, Subtarget);
21880
21881 return FpToInt;
21882 }
21883
21884 // Only handle XLen or i32 types. Other types narrower than XLen will
21885 // eventually be legalized to XLenVT.
21886 if (VT != MVT::i32 && VT != XLenVT)
21887 return SDValue();
21888
21889 unsigned Opc;
21890 if (VT == XLenVT)
21891 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
21892 else
21893 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
21894
21895 SDValue FpToInt = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Src.getOperand(i: 0),
21896 N2: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT));
21897 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FpToInt);
21898}
21899
21900// Fold
21901// (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
21902// (fp_to_int_sat (ftrunc X)) -> (select X == nan, 0, (fcvt X, rtz))
21903// (fp_to_int_sat (ffloor X)) -> (select X == nan, 0, (fcvt X, rdn))
21904// (fp_to_int_sat (fceil X)) -> (select X == nan, 0, (fcvt X, rup))
21905// (fp_to_int_sat (fround X)) -> (select X == nan, 0, (fcvt X, rmm))
21906// (fp_to_int_sat (frint X)) -> (select X == nan, 0, (fcvt X, dyn))
21907static SDValue performFP_TO_INT_SATCombine(SDNode *N,
21908 TargetLowering::DAGCombinerInfo &DCI,
21909 const RISCVSubtarget &Subtarget) {
21910 SelectionDAG &DAG = DCI.DAG;
21911 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
21912 MVT XLenVT = Subtarget.getXLenVT();
21913
21914 // Only handle XLen types. Other types narrower than XLen will eventually be
21915 // legalized to XLenVT.
21916 EVT DstVT = N->getValueType(ResNo: 0);
21917 if (DstVT != XLenVT)
21918 return SDValue();
21919
21920 SDValue Src = N->getOperand(Num: 0);
21921
21922 // Don't do this for strict-fp Src.
21923 if (Src->isStrictFPOpcode())
21924 return SDValue();
21925
21926 // Ensure the FP type is also legal.
21927 if (!TLI.isTypeLegal(VT: Src.getValueType()))
21928 return SDValue();
21929
21930 // Don't do this for f16 with Zfhmin and not Zfh.
21931 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
21932 return SDValue();
21933
21934 EVT SatVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
21935
21936 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Src.getOpcode());
21937 if (FRM == RISCVFPRndMode::Invalid)
21938 return SDValue();
21939
21940 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
21941
21942 unsigned Opc;
21943 if (SatVT == DstVT)
21944 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
21945 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
21946 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
21947 else
21948 return SDValue();
21949 // FIXME: Support other SatVTs by clamping before or after the conversion.
21950
21951 Src = Src.getOperand(i: 0);
21952
21953 SDLoc DL(N);
21954 SDValue FpToInt = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Src,
21955 N2: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT));
21956
21957 // fcvt.wu.* sign extends bit 31 on RV64. FP_TO_UINT_SAT expects to zero
21958 // extend.
21959 if (Opc == RISCVISD::FCVT_WU_RV64)
21960 FpToInt = DAG.getZeroExtendInReg(Op: FpToInt, DL, VT: MVT::i32);
21961
21962 // RISC-V FP-to-int conversions saturate to the destination register size, but
21963 // don't produce 0 for nan.
21964 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: DstVT);
21965 return DAG.getSelectCC(DL, LHS: Src, RHS: Src, True: ZeroInt, False: FpToInt, Cond: ISD::CondCode::SETUO);
21966}
21967
21968// Combine (bitreverse (bswap X)) to the BREV8 GREVI encoding if the type is
21969// smaller than XLenVT.
21970static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG,
21971 const RISCVSubtarget &Subtarget) {
21972 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension");
21973
21974 SDValue Src = N->getOperand(Num: 0);
21975 if (Src.getOpcode() != ISD::BSWAP)
21976 return SDValue();
21977
21978 EVT VT = N->getValueType(ResNo: 0);
21979 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() ||
21980 !llvm::has_single_bit<uint32_t>(Value: VT.getSizeInBits()))
21981 return SDValue();
21982
21983 SDLoc DL(N);
21984 return DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT, Operand: Src.getOperand(i: 0));
21985}
21986
21987/// Matches a reverse shifted right EVL elements, or a vp.reverse.
21988// TODO: Remove vp.reverse
21989static auto m_ReverseEVL = [](auto X, auto EVL) {
21990 using namespace SDPatternMatch;
21991 return m_AnyOf(m_SpliceRight(m_OneUse(m_VectorReverse(X)), m_Poison(), EVL),
21992 m_Node(ISD::EXPERIMENTAL_VP_REVERSE, X, m_Value(), EVL));
21993};
21994
21995// TODO: A vlse.v is not necessarily faster than a vrgather.vv on all uarchs.
21996// Remove once a cost model driven transform is implemented in the loop
21997// vectorizer.
21998static SDValue performReverseEVLCombine(SDNode *N,
21999 TargetLowering::DAGCombinerInfo &DCI,
22000 const RISCVSubtarget &Subtarget) {
22001 SelectionDAG &DAG = DCI.DAG;
22002 // Fold:
22003 // vp.reverse(vp.load(ADDR, REVMASK, EVL), EVL)
22004 // -> vp.strided.load(ADDR, -1, MASK, EVL)
22005 //
22006 // splice.right(reverse(vp.load(ADDR, REVMASK, EVL)), poison, EVL)
22007 // -> vp.strided.load(ADDR, -1, MASK, EVL)
22008 //
22009 // vp.reverse(binop(vp.load(ADDR, REVMASK, EVL), splat), EVL)
22010 // -> binop(vp.strided.load(ADDR, -1, MASK, EVL), splat)
22011 using namespace SDPatternMatch;
22012 SDValue Op, EVL;
22013 if (!sd_match(N, P: m_ReverseEVL(m_Value(N&: Op), m_Value(N&: EVL))))
22014 return SDValue();
22015
22016 VPLoadSDNode *VPLoad = nullptr;
22017 // Find the single vp_load and check all other leaves are splats.
22018 SmallVector<SDValue> Worklist = {Op};
22019 while (!Worklist.empty()) {
22020 SDValue X = Worklist.pop_back_val();
22021 if (DAG.isSplatValue(V: X))
22022 continue;
22023 if (!X.hasOneUser())
22024 return SDValue();
22025 if (auto *VPL = dyn_cast<VPLoadSDNode>(Val&: X)) {
22026 if (VPLoad && VPLoad != VPL)
22027 return SDValue();
22028 VPLoad = VPL;
22029 } else if (DAG.getTargetLoweringInfo().isBinOp(Opcode: X.getOpcode()) &&
22030 X->getNumValues() == 1) {
22031 append_range(C&: Worklist, R: X->op_values());
22032 } else {
22033 return SDValue();
22034 }
22035 }
22036 if (!VPLoad)
22037 return SDValue();
22038
22039 EVT LoadVT = VPLoad->getValueType(ResNo: 0);
22040 // We do not have a strided_load version for masks, and the evl of vp.reverse
22041 // and vp.load should always be the same.
22042 if (!LoadVT.getVectorElementType().isByteSized() ||
22043 EVL != VPLoad->getVectorLength())
22044 return SDValue();
22045
22046 SDValue LoadMask = VPLoad->getMask();
22047 // If Mask is all ones, then load is unmasked and can be reversed.
22048 if (!isOneOrOneSplat(V: LoadMask)) {
22049 // If the mask is not all ones, we can reverse the load if the mask was also
22050 // reversed by a vp.reverse with the same EVL.
22051 SDValue OrigMask;
22052 if (!sd_match(N: LoadMask, P: m_ReverseEVL(m_Value(N&: OrigMask), m_Specific(N: EVL))))
22053 return SDValue();
22054 LoadMask = OrigMask;
22055 }
22056
22057 // Base = LoadAddr + (NumElem - 1) * ElemWidthByte
22058 SDLoc DL(N);
22059 MVT XLenVT = Subtarget.getXLenVT();
22060 SDValue NumElem = VPLoad->getVectorLength();
22061 uint64_t ElemWidthByte = VPLoad->getValueType(ResNo: 0).getScalarSizeInBits() / 8;
22062
22063 SDValue Temp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: NumElem,
22064 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
22065 SDValue Temp2 = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Temp1,
22066 N2: DAG.getConstant(Val: ElemWidthByte, DL, VT: XLenVT));
22067 SDValue Base = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: VPLoad->getBasePtr(), N2: Temp2);
22068 SDValue Stride = DAG.getSignedConstant(Val: -ElemWidthByte, DL, VT: XLenVT);
22069
22070 MachineFunction &MF = DAG.getMachineFunction();
22071 MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
22072 MachineMemOperand *MMO = MF.getMachineMemOperand(
22073 PtrInfo, F: VPLoad->getMemOperand()->getFlags(),
22074 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: VPLoad->getAlign());
22075
22076 SDValue Ret = DAG.getStridedLoadVP(
22077 VT: LoadVT, DL, Chain: VPLoad->getChain(), Ptr: Base, Stride, Mask: LoadMask,
22078 EVL: VPLoad->getVectorLength(), MMO, IsExpanding: VPLoad->isExpandingLoad());
22079
22080 DCI.CombineTo(N: VPLoad, Res0: Ret.getValue(R: 0), Res1: Ret.getValue(R: 1));
22081
22082 // Remove the top level reverse.
22083 (void)sd_match(N, P: m_ReverseEVL(m_Value(N&: Op), m_Value()));
22084 return Op;
22085}
22086
22087// Fold (i32 (bitcast (v4i8/v2i16 const_splat))) to a scalar i32 constant
22088// on RV64.
22089static SDValue performP_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
22090 const RISCVSubtarget &Subtarget) {
22091 SDValue N0 = N->getOperand(Num: 0);
22092 EVT VT = N->getValueType(ResNo: 0);
22093 EVT SrcVT = N0.getValueType();
22094 if (!Subtarget.is64Bit() || VT != MVT::i32 ||
22095 (SrcVT != MVT::v4i8 && SrcVT != MVT::v2i16))
22096 return SDValue();
22097
22098 APInt SplatVal;
22099 if (!ISD::isConstantSplatVector(N: N0.getNode(), SplatValue&: SplatVal))
22100 return SDValue();
22101 return DAG.getConstant(Val: APInt::getSplat(NewLen: VT.getSizeInBits(), V: SplatVal),
22102 DL: SDLoc(N), VT);
22103}
22104
22105static SDValue performVP_STORECombine(SDNode *N, SelectionDAG &DAG,
22106 const RISCVSubtarget &Subtarget) {
22107 // Fold:
22108 // vp.store(vp.reverse(VAL, EVL), ADDR, REVMASK, EVL)
22109 // -> vp.strided.store(VAL, NEW_ADDR, -1, MASK, EVL)
22110 //
22111 // vp.store(splice.right(reverse(VAL), poison, EVL), ADDR, REVMASK, EVL)
22112 // -> vp.strided.store(VAL, NEW_ADDR, -1, MASK, EVL)
22113 auto *VPStore = cast<VPStoreSDNode>(Val: N);
22114 SDValue EVL = VPStore->getVectorLength();
22115
22116 using namespace SDPatternMatch;
22117 SDValue Val;
22118 if (!sd_match(N: VPStore->getValue(),
22119 P: m_OneUse(P: m_ReverseEVL(m_Value(N&: Val), m_Specific(N: EVL)))))
22120 return SDValue();
22121
22122 EVT ReverseVT = VPStore->getValue()->getValueType(ResNo: 0);
22123
22124 // We do not have a strided_store version for masks.
22125 if (!ReverseVT.getVectorElementType().isByteSized())
22126 return SDValue();
22127
22128 SDValue StoreMask = VPStore->getMask();
22129 // If Mask is all ones, then load is unmasked and can be reversed.
22130 if (!isOneOrOneSplat(V: StoreMask)) {
22131 // If the mask is not all ones, we can reverse the store if the mask was
22132 // also reversed by a vp.reverse with the same EVL.
22133 SDValue OrigMask;
22134 if (!sd_match(N: StoreMask, P: m_ReverseEVL(m_Value(N&: OrigMask), m_Specific(N: EVL))))
22135 return SDValue();
22136 StoreMask = OrigMask;
22137 }
22138
22139 // Base = StoreAddr + (NumElem - 1) * ElemWidthByte
22140 SDLoc DL(N);
22141 MVT XLenVT = Subtarget.getXLenVT();
22142 SDValue NumElem = VPStore->getVectorLength();
22143 uint64_t ElemWidthByte = ReverseVT.getScalarSizeInBits() / 8;
22144
22145 SDValue Temp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: NumElem,
22146 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
22147 SDValue Temp2 = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Temp1,
22148 N2: DAG.getConstant(Val: ElemWidthByte, DL, VT: XLenVT));
22149 SDValue Base =
22150 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: VPStore->getBasePtr(), N2: Temp2);
22151 SDValue Stride = DAG.getSignedConstant(Val: -ElemWidthByte, DL, VT: XLenVT);
22152
22153 MachineFunction &MF = DAG.getMachineFunction();
22154 MachinePointerInfo PtrInfo(VPStore->getAddressSpace());
22155 MachineMemOperand *MMO = MF.getMachineMemOperand(
22156 PtrInfo, F: VPStore->getMemOperand()->getFlags(),
22157 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: VPStore->getAlign());
22158
22159 return DAG.getStridedStoreVP(
22160 Chain: VPStore->getChain(), DL, Val, Ptr: Base, Offset: VPStore->getOffset(), Stride,
22161 Mask: StoreMask, EVL: VPStore->getVectorLength(), MemVT: VPStore->getMemoryVT(), MMO,
22162 AM: VPStore->getAddressingMode(), IsTruncating: VPStore->isTruncatingStore(),
22163 IsCompressing: VPStore->isCompressingStore());
22164}
22165
22166/// Given
22167/// ```
22168/// %s = <0, 1, 2, 3, ...>
22169/// %M = splat %m
22170/// %p = setcc ult %s, %M
22171/// %v = mask.load %addr, %p, %passthru
22172/// ```
22173/// we can turn this use a vp.load + vp.merge instead to avoid
22174/// emitting mask. The VL of these two vp operations would be
22175/// `min(%m, <number of vector elements>)`
22176static SDValue performMaskedLoadToVPLoadCombine(MaskedLoadSDNode *MLoad,
22177 SelectionDAG &DAG) {
22178 using namespace SDPatternMatch;
22179 EVT MaskVT = MLoad->getMask().getValueType();
22180 assert(MaskVT.isVector());
22181 if (!MaskVT.isFixedLengthVector())
22182 return SDValue();
22183 unsigned NumElements = MaskVT.getVectorNumElements();
22184 SDLoc DL(MLoad);
22185
22186 SDValue SetCCLHS, SetCCRHS;
22187 ISD::CondCode CC;
22188 if (!sd_match(N: MLoad->getMask(), P: m_SetCC(LHS: m_Value(N&: SetCCLHS), RHS: m_Value(N&: SetCCRHS),
22189 CC: m_CondCode(CC))) ||
22190 SetCCLHS->getOpcode() != ISD::BUILD_VECTOR ||
22191 !(CC == ISD::SETULT || CC == ISD::SETLT) ||
22192 !SetCCLHS.getValueType().isInteger())
22193 return SDValue();
22194 bool IsSigned = ISD::isSignedIntSetCC(Code: CC);
22195
22196 SDValue Boundary = DAG.getSplatValue(V: SetCCRHS, /*LegalizeType=*/LegalTypes: true);
22197 if (!Boundary)
22198 return SDValue();
22199
22200 // Return {a,n} from a build_vector sequence of {a, a+n, a+2n, a+3n, ....}
22201 auto StepVector = cast<BuildVectorSDNode>(Val&: SetCCLHS)->isArithmeticSequence();
22202 if (!StepVector || !StepVector->first.isZero() || !StepVector->second.isOne())
22203 return SDValue();
22204 EVT LenVT = Boundary.getValueType();
22205
22206 SDValue VL = DAG.getNode(Opcode: IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT: LenVT,
22207 N1: Boundary, N2: DAG.getConstant(Val: NumElements, DL, VT: LenVT));
22208
22209 SDValue VPLoad = DAG.getLoadVP(
22210 VT: MLoad->getValueType(ResNo: 0), dl: DL, Chain: MLoad->getChain(), Ptr: MLoad->getBasePtr(),
22211 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL, PtrInfo: MLoad->getPointerInfo(),
22212 Alignment: MLoad->getBaseAlign(), MMOFlags: MLoad->getMemOperand()->getFlags(), AAInfo: AAMDNodes());
22213 if (MLoad->getPassThru().isUndef())
22214 return DAG.getMergeValues(Ops: {VPLoad, VPLoad.getValue(R: 1)}, dl: DL);
22215 // Insert vp.merge if there is a passthru.
22216 SDValue VPMerge = DAG.getNode(Opcode: ISD::VP_MERGE, DL, VT: MLoad->getValueType(ResNo: 0),
22217 N1: DAG.getAllOnesConstant(DL, VT: MaskVT), N2: VPLoad,
22218 N3: MLoad->getPassThru(), N4: VL);
22219 return DAG.getMergeValues(Ops: {VPMerge, VPLoad.getValue(R: 1)}, dl: DL);
22220}
22221
22222static SDValue performVECTOR_INTERLEAVECombine(SDNode *N, SelectionDAG &DAG) {
22223 SDLoc DL(N);
22224 const unsigned Factor = N->getNumOperands();
22225 assert(Factor <= 8);
22226 MVT VecVT = N->getSimpleValueType(ResNo: 0);
22227
22228 if (Factor != 4 && Factor != 8)
22229 return SDValue();
22230
22231 // Interleave by a tree of vzip.vv instructions.
22232 SmallVector<SDValue, 8> Operands(N->op_values());
22233 // First, reorder the operands.
22234 // For Factor=4, given the original operand order `ABCD`, we need
22235 // to reorder it into `ACBD`.
22236 // For Factor=8, given the original operand order `ABCDEFGH`, the new
22237 // order should be `AECGBFDH`.
22238 // So the rule here is that for every operands with an odd index `I`, swap
22239 // it with the operand of index `I + (Factor / 2 - 1)`.
22240 for (unsigned I = 1U, HalfFactor = Factor / 2; I < HalfFactor; I += 2)
22241 std::swap(a&: Operands[I], b&: Operands[I + (HalfFactor - 1)]);
22242
22243 for (unsigned CurrFactor = Factor; CurrFactor > 1; CurrFactor /= 2) {
22244 // Generate a vector_interleave2 for every two operands.
22245 for (unsigned I = 0U; I < CurrFactor; I += 2) {
22246 assert(Operands[I].getValueType() == Operands[I + 1].getValueType());
22247 EVT OperandEVT = Operands[I].getValueType();
22248 EVT ResEVT = OperandEVT.getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
22249
22250 SDValue V = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL,
22251 VTList: DAG.getVTList(VT1: OperandEVT, VT2: OperandEVT),
22252 N1: Operands[I], N2: Operands[I + 1]);
22253 Operands[I / 2] =
22254 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResEVT, N1: V, N2: V.getValue(R: 1));
22255 }
22256 }
22257
22258 // Operands[0] is the resulting concat vector, but we need to split into
22259 // Factor parts.
22260 SDValue Interleaved = Operands[0];
22261 for (unsigned I = 0U; I < Factor; ++I)
22262 Operands[I] = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved,
22263 Idx: I * VecVT.getVectorMinNumElements());
22264
22265 return DAG.getMergeValues(Ops: Operands, dl: DL);
22266}
22267
22268// Convert from one FMA opcode to another based on whether we are negating the
22269// multiply result and/or the accumulator.
22270// NOTE: Only supports RVV operations with VL.
22271static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc) {
22272 // Negating the multiply result changes ADD<->SUB and toggles 'N'.
22273 if (NegMul) {
22274 // clang-format off
22275 switch (Opcode) {
22276 default: llvm_unreachable("Unexpected opcode");
22277 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
22278 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
22279 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
22280 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
22281 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
22282 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
22283 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
22284 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
22285 }
22286 // clang-format on
22287 }
22288
22289 // Negating the accumulator changes ADD<->SUB.
22290 if (NegAcc) {
22291 // clang-format off
22292 switch (Opcode) {
22293 default: llvm_unreachable("Unexpected opcode");
22294 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
22295 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
22296 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
22297 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
22298 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
22299 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
22300 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
22301 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
22302 }
22303 // clang-format on
22304 }
22305
22306 return Opcode;
22307}
22308
22309static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) {
22310 // Fold FNEG_VL into FMA opcodes.
22311 // The first operand of strict-fp is chain.
22312 bool IsStrict =
22313 DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(Opcode: N->getOpcode());
22314 unsigned Offset = IsStrict ? 1 : 0;
22315 SDValue A = N->getOperand(Num: 0 + Offset);
22316 SDValue B = N->getOperand(Num: 1 + Offset);
22317 SDValue C = N->getOperand(Num: 2 + Offset);
22318 SDValue Mask = N->getOperand(Num: 3 + Offset);
22319 SDValue VL = N->getOperand(Num: 4 + Offset);
22320
22321 auto invertIfNegative = [&Mask, &VL](SDValue &V) {
22322 if (V.getOpcode() == RISCVISD::FNEG_VL && V.getOperand(i: 1) == Mask &&
22323 V.getOperand(i: 2) == VL) {
22324 // Return the negated input.
22325 V = V.getOperand(i: 0);
22326 return true;
22327 }
22328
22329 return false;
22330 };
22331
22332 bool NegA = invertIfNegative(A);
22333 bool NegB = invertIfNegative(B);
22334 bool NegC = invertIfNegative(C);
22335
22336 // If no operands are negated, we're done.
22337 if (!NegA && !NegB && !NegC)
22338 return SDValue();
22339
22340 unsigned NewOpcode = negateFMAOpcode(Opcode: N->getOpcode(), NegMul: NegA != NegB, NegAcc: NegC);
22341 if (IsStrict)
22342 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VTList: N->getVTList(),
22343 Ops: {N->getOperand(Num: 0), A, B, C, Mask, VL});
22344 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: A, N2: B, N3: C, N4: Mask,
22345 N5: VL);
22346}
22347
22348static SDValue performVFMADD_VLCombine(SDNode *N,
22349 TargetLowering::DAGCombinerInfo &DCI,
22350 const RISCVSubtarget &Subtarget) {
22351 SelectionDAG &DAG = DCI.DAG;
22352
22353 if (SDValue V = combineVFMADD_VLWithVFNEG_VL(N, DAG))
22354 return V;
22355
22356 // FIXME: Ignore strict opcodes for now.
22357 if (DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(Opcode: N->getOpcode()))
22358 return SDValue();
22359
22360 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
22361}
22362
22363static SDValue performVEXT_VLCombine(SDNode *N,
22364 TargetLowering::DAGCombinerInfo &DCI,
22365 const RISCVSubtarget &Subtarget) {
22366 unsigned Opcode = N->getOpcode();
22367 assert((Opcode == RISCVISD::VSEXT_VL || Opcode == RISCVISD::VZEXT_VL) &&
22368 "Unexpected opcode");
22369
22370 SDValue Inner = N->getOperand(Num: 0);
22371 SDValue Mask = N->getOperand(Num: 1);
22372 SDValue VL = N->getOperand(Num: 2);
22373
22374 // Combine (vext_vl (vext_vl x, m, vl), m, vl) -> (vext_vl x, m, vl)
22375 // where vext_vl is either vsext_vl or vzext_vl.
22376 using namespace SDPatternMatch;
22377 SDValue Src;
22378 if (!sd_match(N: Inner,
22379 P: m_OneUse(P: m_Node(Opcode, preds: m_Value(N&: Src), preds: m_Value(), preds: m_Value()))))
22380 return SDValue();
22381
22382 MVT DstVT = N->getSimpleValueType(ResNo: 0);
22383 return DCI.DAG.getNode(Opcode, DL: SDLoc(N), VT: DstVT, N1: Src, N2: Mask, N3: VL);
22384}
22385
22386static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
22387 const RISCVSubtarget &Subtarget) {
22388 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode");
22389
22390 EVT VT = N->getValueType(ResNo: 0);
22391
22392 if (VT != Subtarget.getXLenVT())
22393 return SDValue();
22394
22395 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
22396 return SDValue();
22397 uint64_t ShAmt = N->getConstantOperandVal(Num: 1);
22398
22399 SDValue N0 = N->getOperand(Num: 0);
22400
22401 // Combine (sra (sext_inreg (shl X, C1), iX), C2) ->
22402 // (sra (shl X, C1+(XLen-iX)), C2+(XLen-iX)) so it gets selected as SLLI+SRAI.
22403 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse()) {
22404 unsigned ExtSize =
22405 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
22406 if (ShAmt < ExtSize && N0.getOperand(i: 0).getOpcode() == ISD::SHL &&
22407 N0.getOperand(i: 0).hasOneUse() &&
22408 isa<ConstantSDNode>(Val: N0.getOperand(i: 0).getOperand(i: 1))) {
22409 uint64_t LShAmt = N0.getOperand(i: 0).getConstantOperandVal(i: 1);
22410 if (LShAmt < ExtSize) {
22411 unsigned Size = VT.getSizeInBits();
22412 SDLoc ShlDL(N0.getOperand(i: 0));
22413 SDValue Shl =
22414 DAG.getNode(Opcode: ISD::SHL, DL: ShlDL, VT, N1: N0.getOperand(i: 0).getOperand(i: 0),
22415 N2: DAG.getConstant(Val: LShAmt + (Size - ExtSize), DL: ShlDL, VT));
22416 SDLoc DL(N);
22417 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Shl,
22418 N2: DAG.getConstant(Val: ShAmt + (Size - ExtSize), DL, VT));
22419 }
22420 }
22421 }
22422
22423 if (ShAmt > 32 || VT != MVT::i64)
22424 return SDValue();
22425
22426 // Combine (sra (shl X, 32), 32 - C) -> (shl (sext_inreg X, i32), C)
22427 // FIXME: Should this be a generic combine? There's a similar combine on X86.
22428 //
22429 // Also try these folds where an add or sub is in the middle.
22430 // (sra (add (shl X, 32), C1), 32 - C) -> (shl (sext_inreg (add X, C1), C)
22431 // (sra (sub C1, (shl X, 32)), 32 - C) -> (shl (sext_inreg (sub C1, X), C)
22432 SDValue Shl;
22433 ConstantSDNode *AddC = nullptr;
22434
22435 // We might have an ADD or SUB between the SRA and SHL.
22436 bool IsAdd = N0.getOpcode() == ISD::ADD;
22437 if ((IsAdd || N0.getOpcode() == ISD::SUB)) {
22438 // Other operand needs to be a constant we can modify.
22439 AddC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: IsAdd ? 1 : 0));
22440 if (!AddC)
22441 return SDValue();
22442
22443 // AddC needs to have at least 32 trailing zeros.
22444 if (llvm::countr_zero(Val: AddC->getZExtValue()) < 32)
22445 return SDValue();
22446
22447 // All users should be a shift by constant less than or equal to 32. This
22448 // ensures we'll do this optimization for each of them to produce an
22449 // add/sub+sext_inreg they can all share.
22450 for (SDNode *U : N0->users()) {
22451 if (U->getOpcode() != ISD::SRA ||
22452 !isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) ||
22453 U->getConstantOperandVal(Num: 1) > 32)
22454 return SDValue();
22455 }
22456
22457 Shl = N0.getOperand(i: IsAdd ? 0 : 1);
22458 } else {
22459 // Not an ADD or SUB.
22460 Shl = N0;
22461 }
22462
22463 // Look for a shift left by 32.
22464 if (Shl.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: Shl.getOperand(i: 1)) ||
22465 Shl.getConstantOperandVal(i: 1) != 32)
22466 return SDValue();
22467
22468 // We if we didn't look through an add/sub, then the shl should have one use.
22469 // If we did look through an add/sub, the sext_inreg we create is free so
22470 // we're only creating 2 new instructions. It's enough to only remove the
22471 // original sra+add/sub.
22472 if (!AddC && !Shl.hasOneUse())
22473 return SDValue();
22474
22475 SDLoc DL(N);
22476 SDValue In = Shl.getOperand(i: 0);
22477
22478 // If we looked through an ADD or SUB, we need to rebuild it with the shifted
22479 // constant.
22480 if (AddC) {
22481 SDValue ShiftedAddC =
22482 DAG.getConstant(Val: AddC->getZExtValue() >> 32, DL, VT: MVT::i64);
22483 if (IsAdd)
22484 In = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: In, N2: ShiftedAddC);
22485 else
22486 In = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: ShiftedAddC, N2: In);
22487 }
22488
22489 SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: In,
22490 N2: DAG.getValueType(MVT::i32));
22491 if (ShAmt == 32)
22492 return SExt;
22493
22494 return DAG.getNode(
22495 Opcode: ISD::SHL, DL, VT: MVT::i64, N1: SExt,
22496 N2: DAG.getConstant(Val: 32 - ShAmt, DL, VT: MVT::i64));
22497}
22498
22499// Invert (and/or (set cc X, Y), (xor Z, 1)) to (or/and (set !cc X, Y)), Z) if
22500// the result is used as the condition of a br_cc or select_cc we can invert,
22501// inverting the setcc is free, and Z is 0/1. Caller will invert the
22502// br_cc/select_cc.
22503static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
22504 bool IsAnd = Cond.getOpcode() == ISD::AND;
22505 if (!IsAnd && Cond.getOpcode() != ISD::OR)
22506 return SDValue();
22507
22508 if (!Cond.hasOneUse())
22509 return SDValue();
22510
22511 SDValue Setcc = Cond.getOperand(i: 0);
22512 SDValue Xor = Cond.getOperand(i: 1);
22513 // Canonicalize setcc to LHS.
22514 if (Setcc.getOpcode() != ISD::SETCC)
22515 std::swap(a&: Setcc, b&: Xor);
22516 // LHS should be a setcc and RHS should be an xor.
22517 if (Setcc.getOpcode() != ISD::SETCC || !Setcc.hasOneUse() ||
22518 Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
22519 return SDValue();
22520
22521 // If the condition is an And, SimplifyDemandedBits may have changed
22522 // (xor Z, 1) to (not Z).
22523 SDValue Xor1 = Xor.getOperand(i: 1);
22524 if (!isOneConstant(V: Xor1) && !(IsAnd && isAllOnesConstant(V: Xor1)))
22525 return SDValue();
22526
22527 EVT VT = Cond.getValueType();
22528 SDValue Xor0 = Xor.getOperand(i: 0);
22529
22530 // The LHS of the xor needs to be 0/1.
22531 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
22532 if (!DAG.MaskedValueIsZero(Op: Xor0, Mask))
22533 return SDValue();
22534
22535 // We can only invert integer setccs.
22536 EVT SetCCOpVT = Setcc.getOperand(i: 0).getValueType();
22537 if (!SetCCOpVT.isScalarInteger())
22538 return SDValue();
22539
22540 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Setcc.getOperand(i: 2))->get();
22541 if (ISD::isIntEqualitySetCC(Code: CCVal)) {
22542 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
22543 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT, LHS: Setcc.getOperand(i: 0),
22544 RHS: Setcc.getOperand(i: 1), Cond: CCVal);
22545 } else if (CCVal == ISD::SETLT && isNullConstant(V: Setcc.getOperand(i: 0))) {
22546 // Invert (setlt 0, X) by converting to (setlt X, 1).
22547 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT, LHS: Setcc.getOperand(i: 1),
22548 RHS: DAG.getConstant(Val: 1, DL: SDLoc(Setcc), VT), Cond: CCVal);
22549 } else if (CCVal == ISD::SETLT && isOneConstant(V: Setcc.getOperand(i: 1))) {
22550 // (setlt X, 1) by converting to (setlt 0, X).
22551 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT,
22552 LHS: DAG.getConstant(Val: 0, DL: SDLoc(Setcc), VT),
22553 RHS: Setcc.getOperand(i: 0), Cond: CCVal);
22554 } else
22555 return SDValue();
22556
22557 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
22558 return DAG.getNode(Opcode: Opc, DL: SDLoc(Cond), VT, N1: Setcc, N2: Xor.getOperand(i: 0));
22559}
22560
22561// Perform common combines for BR_CC and SELECT_CC conditions.
22562static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
22563 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
22564 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
22565
22566 // As far as arithmetic right shift always saves the sign,
22567 // shift can be omitted.
22568 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
22569 // setge (sra X, N), 0 -> setge X, 0
22570 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
22571 LHS.getOpcode() == ISD::SRA) {
22572 LHS = LHS.getOperand(i: 0);
22573 return true;
22574 }
22575
22576 if (!ISD::isIntEqualitySetCC(Code: CCVal))
22577 return false;
22578
22579 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
22580 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
22581 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
22582 LHS.getOperand(i: 0).getValueType() == Subtarget.getXLenVT()) {
22583 // If we're looking for eq 0 instead of ne 0, we need to invert the
22584 // condition.
22585 bool Invert = CCVal == ISD::SETEQ;
22586 CCVal = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
22587 if (Invert)
22588 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
22589
22590 RHS = LHS.getOperand(i: 1);
22591 LHS = LHS.getOperand(i: 0);
22592 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
22593
22594 CC = DAG.getCondCode(Cond: CCVal);
22595 return true;
22596 }
22597
22598 auto isFoldableXorEq = [&DAG](SDValue LHS, SDValue RHS) -> bool {
22599 if (LHS.getOpcode() != ISD::XOR || !isNullConstant(V: RHS))
22600 return false;
22601
22602 // If XOR cannot be an XORI, allow the fold.
22603 const auto *XorCnst = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
22604 if (!XorCnst || !isInt<12>(x: XorCnst->getSExtValue()))
22605 return true;
22606
22607 // Fold (X(i1) ^ 1) == 0 -> X != 0
22608 SDValue VarOp = LHS.getOperand(i: 0);
22609 const APInt Mask = APInt::getBitsSetFrom(numBits: VarOp.getValueSizeInBits(), loBit: 1);
22610 if (XorCnst->getSExtValue() == 1 && DAG.MaskedValueIsZero(Op: VarOp, Mask))
22611 return true;
22612
22613 // If the Xor is only used by select or br_cc, allow the fold.
22614 return all_of(Range: LHS->users(), P: [](const SDNode *UserNode) {
22615 const unsigned Opcode = UserNode->getOpcode();
22616 return Opcode == RISCVISD::SELECT_CC || Opcode == RISCVISD::BR_CC;
22617 });
22618 };
22619 // Fold ((xor X, Y), 0, eq/ne) -> (X, Y, eq/ne)
22620 if (isFoldableXorEq(LHS, RHS)) {
22621 RHS = LHS.getOperand(i: 1);
22622 LHS = LHS.getOperand(i: 0);
22623 return true;
22624 }
22625 // Fold ((sext (xor X, C)), 0, eq/ne) -> ((sext(X), C, eq/ne)
22626 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG) {
22627 const SDValue LHS0 = LHS.getOperand(i: 0);
22628 if (isFoldableXorEq(LHS0, RHS) && isa<ConstantSDNode>(Val: LHS0.getOperand(i: 1))) {
22629 // SEXT(XOR(X, Y)) -> XOR(SEXT(X), SEXT(Y)))
22630 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: LHS.getValueType(),
22631 N1: LHS0.getOperand(i: 1), N2: LHS.getOperand(i: 1));
22632 LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: LHS.getValueType(),
22633 N1: LHS0.getOperand(i: 0), N2: LHS.getOperand(i: 1));
22634 return true;
22635 }
22636 }
22637
22638 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, XLen-1-C), 0, ge/lt)
22639 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
22640 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
22641 SDValue LHS0 = LHS.getOperand(i: 0);
22642 if (LHS0.getOpcode() == ISD::AND &&
22643 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
22644 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
22645 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
22646 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
22647 // XAndesPerf supports branch on test bit.
22648 if (Subtarget.hasVendorXAndesPerf()) {
22649 LHS =
22650 DAG.getNode(Opcode: ISD::AND, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
22651 N2: DAG.getConstant(Val: Mask, DL, VT: LHS.getValueType()));
22652 return true;
22653 }
22654
22655 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
22656 CC = DAG.getCondCode(Cond: CCVal);
22657
22658 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
22659 LHS = LHS0.getOperand(i: 0);
22660 if (ShAmt != 0)
22661 LHS =
22662 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
22663 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
22664 return true;
22665 }
22666 }
22667 }
22668
22669 // (X, 1, setne) -> // (X, 0, seteq) if we can prove X is 0/1.
22670 // This can occur when legalizing some floating point comparisons.
22671 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
22672 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
22673 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
22674 CC = DAG.getCondCode(Cond: CCVal);
22675 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
22676 return true;
22677 }
22678
22679 if (isNullConstant(V: RHS)) {
22680 if (SDValue NewCond = tryDemorganOfBooleanCondition(Cond: LHS, DAG)) {
22681 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
22682 CC = DAG.getCondCode(Cond: CCVal);
22683 LHS = NewCond;
22684 return true;
22685 }
22686 }
22687
22688 return false;
22689}
22690
22691// Fold
22692// (select C, (add Y, X), Y) -> (add Y, (select C, X, 0)).
22693// (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
22694// (select C, (or Y, X), Y) -> (or Y, (select C, X, 0)).
22695// (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
22696// (select C, (rotl Y, X), Y) -> (rotl Y, (select C, X, 0)).
22697// (select C, (rotr Y, X), Y) -> (rotr Y, (select C, X, 0)).
22698static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
22699 SDValue TrueVal, SDValue FalseVal,
22700 bool Swapped) {
22701 bool Commutative = true;
22702 unsigned Opc = TrueVal.getOpcode();
22703 switch (Opc) {
22704 default:
22705 return SDValue();
22706 case ISD::SHL:
22707 case ISD::SRA:
22708 case ISD::SRL:
22709 case ISD::SUB:
22710 case ISD::ROTL:
22711 case ISD::ROTR:
22712 Commutative = false;
22713 break;
22714 case ISD::ADD:
22715 case ISD::OR:
22716 case ISD::XOR:
22717 case ISD::UMIN:
22718 case ISD::UMAX:
22719 break;
22720 }
22721
22722 if (!TrueVal.hasOneUse())
22723 return SDValue();
22724
22725 unsigned OpToFold;
22726 if (FalseVal == TrueVal.getOperand(i: 0))
22727 OpToFold = 0;
22728 else if (Commutative && FalseVal == TrueVal.getOperand(i: 1))
22729 OpToFold = 1;
22730 else
22731 return SDValue();
22732
22733 EVT VT = N->getValueType(ResNo: 0);
22734 SDLoc DL(N);
22735 SDValue OtherOp = TrueVal.getOperand(i: 1 - OpToFold);
22736 EVT OtherOpVT = OtherOp.getValueType();
22737 SDValue IdentityOperand =
22738 DAG.getIdentityElement(Opcode: Opc, DL, VT: OtherOpVT, Flags: N->getFlags());
22739 if (!Commutative)
22740 IdentityOperand = DAG.getConstant(Val: 0, DL, VT: OtherOpVT);
22741 assert(IdentityOperand && "No identity operand!");
22742
22743 if (Swapped)
22744 std::swap(a&: OtherOp, b&: IdentityOperand);
22745 SDValue NewSel =
22746 DAG.getSelect(DL, VT: OtherOpVT, Cond: N->getOperand(Num: 0), LHS: OtherOp, RHS: IdentityOperand);
22747 return DAG.getNode(Opcode: TrueVal.getOpcode(), DL, VT, N1: FalseVal, N2: NewSel);
22748}
22749
22750// This tries to get rid of `select` and `icmp` that are being used to handle
22751// `Targets` that do not support `cttz(0)`/`ctlz(0)`.
22752static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG) {
22753 SDValue Cond = N->getOperand(Num: 0);
22754
22755 // This represents either CTTZ or CTLZ instruction.
22756 SDValue CountZeroes;
22757
22758 SDValue ValOnZero;
22759
22760 if (Cond.getOpcode() != ISD::SETCC)
22761 return SDValue();
22762
22763 if (!isNullConstant(V: Cond->getOperand(Num: 1)))
22764 return SDValue();
22765
22766 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond->getOperand(Num: 2))->get();
22767 if (CCVal == ISD::CondCode::SETEQ) {
22768 CountZeroes = N->getOperand(Num: 2);
22769 ValOnZero = N->getOperand(Num: 1);
22770 } else if (CCVal == ISD::CondCode::SETNE) {
22771 CountZeroes = N->getOperand(Num: 1);
22772 ValOnZero = N->getOperand(Num: 2);
22773 } else {
22774 return SDValue();
22775 }
22776
22777 if (CountZeroes.getOpcode() == ISD::TRUNCATE ||
22778 CountZeroes.getOpcode() == ISD::ZERO_EXTEND)
22779 CountZeroes = CountZeroes.getOperand(i: 0);
22780
22781 if (CountZeroes.getOpcode() != ISD::CTTZ &&
22782 CountZeroes.getOpcode() != ISD::CTTZ_ZERO_POISON &&
22783 CountZeroes.getOpcode() != ISD::CTLZ &&
22784 CountZeroes.getOpcode() != ISD::CTLZ_ZERO_POISON)
22785 return SDValue();
22786
22787 if (!isNullConstant(V: ValOnZero))
22788 return SDValue();
22789
22790 SDValue CountZeroesArgument = CountZeroes->getOperand(Num: 0);
22791 if (Cond->getOperand(Num: 0) != CountZeroesArgument)
22792 return SDValue();
22793
22794 unsigned BitWidth = CountZeroes.getValueSizeInBits();
22795 if (!isPowerOf2_32(Value: BitWidth))
22796 return SDValue();
22797
22798 if (CountZeroes.getOpcode() == ISD::CTTZ_ZERO_POISON) {
22799 CountZeroes = DAG.getNode(Opcode: ISD::CTTZ, DL: SDLoc(CountZeroes),
22800 VT: CountZeroes.getValueType(), Operand: CountZeroesArgument);
22801 } else if (CountZeroes.getOpcode() == ISD::CTLZ_ZERO_POISON) {
22802 CountZeroes = DAG.getNode(Opcode: ISD::CTLZ, DL: SDLoc(CountZeroes),
22803 VT: CountZeroes.getValueType(), Operand: CountZeroesArgument);
22804 }
22805
22806 SDValue BitWidthMinusOne =
22807 DAG.getConstant(Val: BitWidth - 1, DL: SDLoc(N), VT: CountZeroes.getValueType());
22808
22809 auto AndNode = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CountZeroes.getValueType(),
22810 N1: CountZeroes, N2: BitWidthMinusOne);
22811 return DAG.getZExtOrTrunc(Op: AndNode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
22812}
22813
22814static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG,
22815 const RISCVSubtarget &Subtarget) {
22816 SDValue Cond = N->getOperand(Num: 0);
22817 SDValue True = N->getOperand(Num: 1);
22818 SDValue False = N->getOperand(Num: 2);
22819 SDLoc DL(N);
22820 EVT VT = N->getValueType(ResNo: 0);
22821 EVT CondVT = Cond.getValueType();
22822
22823 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
22824 return SDValue();
22825
22826 // Replace (setcc eq (and x, C)) with (setcc ne (and x, C))) to generate
22827 // BEXTI, where C is power of 2.
22828 if (Subtarget.hasBEXTILike() && VT.isScalarInteger() &&
22829 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXTHeadCondMov())) {
22830 SDValue LHS = Cond.getOperand(i: 0);
22831 SDValue RHS = Cond.getOperand(i: 1);
22832 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
22833 if (CC == ISD::SETEQ && LHS.getOpcode() == ISD::AND &&
22834 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) && isNullConstant(V: RHS)) {
22835 const APInt &MaskVal = LHS.getConstantOperandAPInt(i: 1);
22836 if (MaskVal.isPowerOf2() && !MaskVal.isSignedIntN(N: 12))
22837 return DAG.getSelect(DL, VT,
22838 Cond: DAG.getSetCC(DL, VT: CondVT, LHS, RHS, Cond: ISD::SETNE),
22839 LHS: False, RHS: True);
22840 }
22841 }
22842 return SDValue();
22843}
22844
22845static SDValue
22846canonicalizeVSelectTrueToOneUse(SDNode *N, SelectionDAG &DAG,
22847 const RISCVSubtarget &Subtarget) {
22848 SDValue CC = N->getOperand(Num: 0);
22849 SDValue TrueVal = N->getOperand(Num: 1);
22850 SDValue FalseVal = N->getOperand(Num: 2);
22851
22852 if (CC.getOpcode() != ISD::SETCC || !CC.hasOneUse() || TrueVal.hasOneUse() ||
22853 !FalseVal.hasOneUse())
22854 return SDValue();
22855
22856 // Only handles ISD::SETEQ and ISD::SETNE; no extra RVV introduced.
22857 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CC.getOperand(i: 2))->get();
22858 if (!isIntEqualitySetCC(Code: CCVal))
22859 return SDValue();
22860
22861 if (DAG.isSplatValue(V: TrueVal) || DAG.isSplatValue(V: FalseVal) ||
22862 TrueVal.getOpcode() == ISD::SPLAT_VECTOR_PARTS ||
22863 FalseVal.getOpcode() == ISD::SPLAT_VECTOR_PARTS ||
22864 TrueVal.getOpcode() == RISCVISD::VMV_V_X_VL ||
22865 FalseVal.getOpcode() == RISCVISD::VMV_V_X_VL)
22866 return SDValue();
22867
22868 SDLoc DL(N);
22869 EVT CVT = CC.getValueType();
22870 SDValue InvertedCC = DAG.getSetCC(DL, VT: CVT, LHS: CC.getOperand(i: 0), RHS: CC.getOperand(i: 1),
22871 Cond: ISD::getSetCCInverse(Operation: CCVal, Type: CVT));
22872 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: N->getValueType(ResNo: 0), N1: InvertedCC, N2: FalseVal,
22873 N3: TrueVal);
22874}
22875
22876static bool matchSelectAddSub(SDValue TrueVal, SDValue FalseVal, bool &SwapCC) {
22877 if (!TrueVal.hasOneUse() || !FalseVal.hasOneUse())
22878 return false;
22879
22880 SwapCC = false;
22881 if (TrueVal.getOpcode() == ISD::SUB && FalseVal.getOpcode() == ISD::ADD) {
22882 std::swap(a&: TrueVal, b&: FalseVal);
22883 SwapCC = true;
22884 }
22885
22886 if (TrueVal.getOpcode() != ISD::ADD || FalseVal.getOpcode() != ISD::SUB)
22887 return false;
22888
22889 SDValue A = FalseVal.getOperand(i: 0);
22890 SDValue B = FalseVal.getOperand(i: 1);
22891 // Add is commutative, so check both orders
22892 return ((TrueVal.getOperand(i: 0) == A && TrueVal.getOperand(i: 1) == B) ||
22893 (TrueVal.getOperand(i: 1) == A && TrueVal.getOperand(i: 0) == B));
22894}
22895
22896static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
22897 const RISCVSubtarget &Subtarget) {
22898 SDLoc DL(N);
22899 EVT VT = N->getValueType(ResNo: 0);
22900 SDValue CC = N->getOperand(Num: 0);
22901 SDValue TrueVal = N->getOperand(Num: 1);
22902 SDValue FalseVal = N->getOperand(Num: 2);
22903
22904 // Convert (vselect CC, true, false) to (vselect InvertCC, false, true) when
22905 // false has one use and true has multiple use.
22906 // It relies on RISCVVectorPeephole.cpp foldVMergeToMask to eliminate
22907 // vmerge.vv
22908 if (SDValue V = canonicalizeVSelectTrueToOneUse(N, DAG, Subtarget))
22909 return V;
22910
22911 // Convert vselect CC, (add a, b), (sub a, b) to add a, (vselect CC, -b, b).
22912 // This allows us match a vadd.vv fed by a masked vrsub, which reduces
22913 // register pressure over the add followed by masked vsub sequence.
22914 bool SwapCC;
22915 if (!matchSelectAddSub(TrueVal, FalseVal, SwapCC))
22916 return SDValue();
22917
22918 SDValue Sub = SwapCC ? TrueVal : FalseVal;
22919 SDValue A = Sub.getOperand(i: 0);
22920 SDValue B = Sub.getOperand(i: 1);
22921
22922 // Arrange the select such that we can match a masked
22923 // vrsub.vi to perform the conditional negate
22924 SDValue NegB = DAG.getNegative(Val: B, DL, VT);
22925 if (!SwapCC)
22926 CC = DAG.getLogicalNOT(DL, Val: CC, VT: CC->getValueType(ResNo: 0));
22927 SDValue NewB = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CC, N2: NegB, N3: B);
22928 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: NewB);
22929}
22930
22931// Fold (iN (select (src >u ((1 << N) - 1)), sext(src >s -1), trunc(src))) to
22932// USATI. This pattern saturates a signed value to an unsigned N-bit range
22933// [0, 2^N-1]:
22934// - If src < 0: result = 0
22935// (via the inner comparison src > -1 = false, sext to 0)
22936// - If src > ((1 << C) - 1): result = all 1s
22937// (via sext(true) = -1 = 0xFF...)
22938// - Otherwise: result = src (via trunc(src))
22939// The outer comparison is unsigned, so negative values appear as large
22940// unsigned values and trigger the saturation to MaxVal path, where the
22941// inner signed comparison then produces 0.
22942// TODO: Support (select (src <=u ((1 << C) - 1)), trunc(src), sext(src >s -1)).
22943static SDValue foldSelectToUSATI(SDNode *N, SelectionDAG &DAG,
22944 const RISCVSubtarget &Subtarget) {
22945 if (!Subtarget.hasStdExtP())
22946 return SDValue();
22947
22948 EVT VT = N->getValueType(ResNo: 0);
22949 MVT XLenVT = Subtarget.getXLenVT();
22950
22951 // Only support scalar integer types smaller than XLenVT
22952 if (!VT.isScalarInteger() || VT.bitsGE(VT: XLenVT))
22953 return SDValue();
22954
22955 unsigned SatWidth = VT.getSizeInBits();
22956 uint64_t MaxVal = (1ULL << SatWidth) - 1;
22957
22958 using namespace SDPatternMatch;
22959
22960 SDValue Src, InnerSetCC, FalseSrc;
22961 if (!sd_match(N, P: m_Select(Cond: m_SetCC(LHS: m_Value(N&: Src), RHS: m_SpecificInt(V: MaxVal),
22962 CC: m_SpecificCondCode(CC: ISD::SETUGT)),
22963 T: m_SExt(Op: m_Value(N&: InnerSetCC)),
22964 F: m_Trunc(Op: m_Value(N&: FalseSrc)))))
22965 return SDValue();
22966
22967 // Src can't be larger than XLenVT.
22968 if (Src.getValueType().bitsGT(VT: XLenVT))
22969 return SDValue();
22970
22971 // Check inner setcc: src > -1 (signed comparison)
22972 if (!sd_match(N: InnerSetCC,
22973 P: m_SpecificVT(RefVT: MVT::i1, P: m_SetCC(LHS: m_Specific(N: Src), RHS: m_AllOnes(),
22974 CC: m_SpecificCondCode(CC: ISD::SETGT)))))
22975 return SDValue();
22976
22977 // It's possible that the input to the setccs is also a truncate, in that
22978 // case the input to the truncate on the select's false operand may be the
22979 // same as the input to this setcc truncate. We need to look through the
22980 // setcc truncate to make sure CmpSrc and FalseSrc come from the same value.
22981 SDValue CmpSrc = Src;
22982 if (CmpSrc != FalseSrc && CmpSrc.getOpcode() == ISD::TRUNCATE)
22983 CmpSrc = CmpSrc.getOperand(i: 0);
22984
22985 if (CmpSrc != FalseSrc)
22986 return SDValue();
22987
22988 // We found a USATI pattern.
22989 SDLoc DL(N);
22990 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: XLenVT, Operand: Src);
22991 SDValue USATI = DAG.getNode(Opcode: RISCVISD::USATI, DL, VT: XLenVT, N1: Src,
22992 N2: DAG.getTargetConstant(Val: SatWidth, DL, VT: XLenVT));
22993 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: USATI);
22994}
22995
22996static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
22997 const RISCVSubtarget &Subtarget) {
22998 if (SDValue Folded = foldSelectOfCTTZOrCTLZ(N, DAG))
22999 return Folded;
23000
23001 if (SDValue V = foldSelectToUSATI(N, DAG, Subtarget))
23002 return V;
23003
23004 if (SDValue V = useInversedSetcc(N, DAG, Subtarget))
23005 return V;
23006
23007 if (Subtarget.hasConditionalMoveFusion())
23008 return SDValue();
23009
23010 SDValue TrueVal = N->getOperand(Num: 1);
23011 SDValue FalseVal = N->getOperand(Num: 2);
23012 if (SDValue V = tryFoldSelectIntoOp(N, DAG, TrueVal, FalseVal, /*Swapped*/false))
23013 return V;
23014 return tryFoldSelectIntoOp(N, DAG, TrueVal: FalseVal, FalseVal: TrueVal, /*Swapped*/true);
23015}
23016
23017/// If we have a build_vector where each lane is binop X, C, where C
23018/// is a constant (but not necessarily the same constant on all lanes),
23019/// form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).
23020/// We assume that materializing a constant build vector will be no more
23021/// expensive that performing O(n) binops.
23022static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
23023 const RISCVSubtarget &Subtarget,
23024 const RISCVTargetLowering &TLI) {
23025 SDLoc DL(N);
23026 EVT VT = N->getValueType(ResNo: 0);
23027
23028 assert(!VT.isScalableVector() && "unexpected build vector");
23029
23030 if (VT.getVectorNumElements() == 1)
23031 return SDValue();
23032
23033 const unsigned Opcode = N->op_begin()->getNode()->getOpcode();
23034 if (!TLI.isBinOp(Opcode))
23035 return SDValue();
23036
23037 if (!TLI.isOperationLegalOrCustom(Op: Opcode, VT) || !TLI.isTypeLegal(VT))
23038 return SDValue();
23039
23040 // This BUILD_VECTOR involves an implicit truncation, and sinking
23041 // truncates through binops is non-trivial.
23042 if (N->op_begin()->getValueType() != VT.getVectorElementType())
23043 return SDValue();
23044
23045 SmallVector<SDValue> LHSOps;
23046 SmallVector<SDValue> RHSOps;
23047 for (SDValue Op : N->ops()) {
23048 if (Op.isUndef()) {
23049 // We can't form a divide or remainder from undef.
23050 if (!DAG.isSafeToSpeculativelyExecute(Opcode))
23051 return SDValue();
23052
23053 LHSOps.push_back(Elt: Op);
23054 RHSOps.push_back(Elt: Op);
23055 continue;
23056 }
23057
23058 // TODO: We can handle operations which have an neutral rhs value
23059 // (e.g. x + 0, a * 1 or a << 0), but we then have to keep track
23060 // of profit in a more explicit manner.
23061 if (Op.getOpcode() != Opcode || !Op.hasOneUse())
23062 return SDValue();
23063
23064 LHSOps.push_back(Elt: Op.getOperand(i: 0));
23065 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
23066 !isa<ConstantFPSDNode>(Val: Op.getOperand(i: 1)))
23067 return SDValue();
23068 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
23069 // have different LHS and RHS types.
23070 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
23071 return SDValue();
23072
23073 RHSOps.push_back(Elt: Op.getOperand(i: 1));
23074 }
23075
23076 return DAG.getNode(Opcode, DL, VT, N1: DAG.getBuildVector(VT, DL, Ops: LHSOps),
23077 N2: DAG.getBuildVector(VT, DL, Ops: RHSOps));
23078}
23079
23080static MVT getQDOTXResultType(MVT OpVT) {
23081 ElementCount OpEC = OpVT.getVectorElementCount();
23082 assert(OpEC.isKnownMultipleOf(4) && OpVT.getVectorElementType() == MVT::i8);
23083 return MVT::getVectorVT(VT: MVT::i32, EC: OpEC.divideCoefficientBy(RHS: 4));
23084}
23085
23086/// Given fixed length vectors A and B with equal element types, but possibly
23087/// different number of elements, return A + B where either A or B is zero
23088/// padded to the larger number of elements.
23089static SDValue getZeroPaddedAdd(const SDLoc &DL, SDValue A, SDValue B,
23090 SelectionDAG &DAG) {
23091 // NOTE: Manually doing the extract/add/insert scheme produces
23092 // significantly better codegen than the naive pad with zeros
23093 // and add scheme.
23094 EVT AVT = A.getValueType();
23095 EVT BVT = B.getValueType();
23096 assert(AVT.getVectorElementType() == BVT.getVectorElementType());
23097 if (AVT.getVectorMinNumElements() > BVT.getVectorMinNumElements()) {
23098 std::swap(a&: A, b&: B);
23099 std::swap(a&: AVT, b&: BVT);
23100 }
23101
23102 SDValue BPart = DAG.getExtractSubvector(DL, VT: AVT, Vec: B, Idx: 0);
23103 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: AVT, N1: A, N2: BPart);
23104 return DAG.getInsertSubvector(DL, Vec: B, SubVec: Res, Idx: 0);
23105}
23106
23107static SDValue foldReduceOperandViaVDOT4A(SDValue InVec, const SDLoc &DL,
23108 SelectionDAG &DAG,
23109 const RISCVSubtarget &Subtarget,
23110 const RISCVTargetLowering &TLI) {
23111 using namespace SDPatternMatch;
23112 // Note: We intentionally do not check the legality of the reduction type.
23113 // We want to handle the m4/m8 *src* types, and thus need to let illegal
23114 // intermediate types flow through here.
23115 if (InVec.getValueType().getVectorElementType() != MVT::i32 ||
23116 !InVec.getValueType().getVectorElementCount().isKnownMultipleOf(RHS: 4))
23117 return SDValue();
23118
23119 // Recurse through adds/disjoint ors (since generic dag canonicalizes to that
23120 // form).
23121 SDValue A, B;
23122 if (sd_match(N: InVec, P: m_AddLike(L: m_Value(N&: A), R: m_Value(N&: B)))) {
23123 SDValue AOpt = foldReduceOperandViaVDOT4A(InVec: A, DL, DAG, Subtarget, TLI);
23124 SDValue BOpt = foldReduceOperandViaVDOT4A(InVec: B, DL, DAG, Subtarget, TLI);
23125 if (AOpt || BOpt) {
23126 if (AOpt)
23127 A = AOpt;
23128 if (BOpt)
23129 B = BOpt;
23130 // From here, we're doing A + B with mixed types, implicitly zero
23131 // padded to the wider type. Note that we *don't* need the result
23132 // type to be the original VT, and in fact prefer narrower ones
23133 // if possible.
23134 return getZeroPaddedAdd(DL, A, B, DAG);
23135 }
23136 }
23137
23138 // zext a <--> partial_reduce_umla 0, a, 1
23139 // sext a <--> partial_reduce_smla 0, a, 1
23140 if (InVec.getOpcode() == ISD::ZERO_EXTEND ||
23141 InVec.getOpcode() == ISD::SIGN_EXTEND) {
23142 SDValue A = InVec.getOperand(i: 0);
23143 EVT OpVT = A.getValueType();
23144 if (OpVT.getVectorElementType() != MVT::i8 || !TLI.isTypeLegal(VT: OpVT))
23145 return SDValue();
23146
23147 MVT ResVT = getQDOTXResultType(OpVT: A.getSimpleValueType());
23148 SDValue B = DAG.getConstant(Val: 0x1, DL, VT: OpVT);
23149 bool IsSigned = InVec.getOpcode() == ISD::SIGN_EXTEND;
23150 unsigned Opc =
23151 IsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
23152 return DAG.getNode(Opcode: Opc, DL, VT: ResVT, Ops: {DAG.getConstant(Val: 0, DL, VT: ResVT), A, B});
23153 }
23154
23155 // mul (sext a, sext b) -> partial_reduce_smla 0, a, b
23156 // mul (zext a, zext b) -> partial_reduce_umla 0, a, b
23157 // mul (sext a, zext b) -> partial_reduce_ssmla 0, a, b
23158 // mul (zext a, sext b) -> partial_reduce_smla 0, b, a (swapped)
23159 if (!sd_match(N: InVec, P: m_Mul(L: m_Value(N&: A), R: m_Value(N&: B))))
23160 return SDValue();
23161
23162 if (!ISD::isExtOpcode(Opcode: A.getOpcode()))
23163 return SDValue();
23164
23165 EVT OpVT = A.getOperand(i: 0).getValueType();
23166 if (OpVT.getVectorElementType() != MVT::i8 ||
23167 OpVT != B.getOperand(i: 0).getValueType() ||
23168 !TLI.isTypeLegal(VT: A.getValueType()))
23169 return SDValue();
23170
23171 unsigned Opc;
23172 if (A.getOpcode() == ISD::SIGN_EXTEND && B.getOpcode() == ISD::SIGN_EXTEND)
23173 Opc = ISD::PARTIAL_REDUCE_SMLA;
23174 else if (A.getOpcode() == ISD::ZERO_EXTEND &&
23175 B.getOpcode() == ISD::ZERO_EXTEND)
23176 Opc = ISD::PARTIAL_REDUCE_UMLA;
23177 else if (A.getOpcode() == ISD::SIGN_EXTEND &&
23178 B.getOpcode() == ISD::ZERO_EXTEND)
23179 Opc = ISD::PARTIAL_REDUCE_SUMLA;
23180 else if (A.getOpcode() == ISD::ZERO_EXTEND &&
23181 B.getOpcode() == ISD::SIGN_EXTEND) {
23182 Opc = ISD::PARTIAL_REDUCE_SUMLA;
23183 std::swap(a&: A, b&: B);
23184 } else
23185 return SDValue();
23186
23187 MVT ResVT = getQDOTXResultType(OpVT: OpVT.getSimpleVT());
23188 return DAG.getNode(
23189 Opcode: Opc, DL, VT: ResVT,
23190 Ops: {DAG.getConstant(Val: 0, DL, VT: ResVT), A.getOperand(i: 0), B.getOperand(i: 0)});
23191}
23192
23193static SDValue performVECREDUCECombine(SDNode *N, SelectionDAG &DAG,
23194 const RISCVSubtarget &Subtarget,
23195 const RISCVTargetLowering &TLI) {
23196 if (!Subtarget.hasStdExtZvdot4a8i())
23197 return SDValue();
23198
23199 SDLoc DL(N);
23200 EVT VT = N->getValueType(ResNo: 0);
23201 SDValue InVec = N->getOperand(Num: 0);
23202 if (SDValue V = foldReduceOperandViaVDOT4A(InVec, DL, DAG, Subtarget, TLI))
23203 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT, Operand: V);
23204 return SDValue();
23205}
23206
23207static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
23208 const RISCVSubtarget &Subtarget,
23209 const RISCVTargetLowering &TLI) {
23210 SDValue InVec = N->getOperand(Num: 0);
23211 SDValue InVal = N->getOperand(Num: 1);
23212 SDValue EltNo = N->getOperand(Num: 2);
23213 SDLoc DL(N);
23214
23215 EVT VT = InVec.getValueType();
23216 if (VT.isScalableVector())
23217 return SDValue();
23218
23219 if (!InVec.hasOneUse())
23220 return SDValue();
23221
23222 // Given insert_vector_elt (binop a, VecC), (same_binop b, C2), Elt
23223 // move the insert_vector_elts into the arms of the binop. Note that
23224 // the new RHS must be a constant.
23225 const unsigned InVecOpcode = InVec->getOpcode();
23226 if (InVecOpcode == InVal->getOpcode() && TLI.isBinOp(Opcode: InVecOpcode) &&
23227 InVal.hasOneUse()) {
23228 SDValue InVecLHS = InVec->getOperand(Num: 0);
23229 SDValue InVecRHS = InVec->getOperand(Num: 1);
23230 SDValue InValLHS = InVal->getOperand(Num: 0);
23231 SDValue InValRHS = InVal->getOperand(Num: 1);
23232
23233 if (!ISD::isBuildVectorOfConstantSDNodes(N: InVecRHS.getNode()))
23234 return SDValue();
23235 if (!isa<ConstantSDNode>(Val: InValRHS) && !isa<ConstantFPSDNode>(Val: InValRHS))
23236 return SDValue();
23237 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
23238 // have different LHS and RHS types.
23239 if (InVec.getOperand(i: 0).getValueType() != InVec.getOperand(i: 1).getValueType())
23240 return SDValue();
23241 SDValue LHS = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
23242 N1: InVecLHS, N2: InValLHS, N3: EltNo);
23243 SDValue RHS = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
23244 N1: InVecRHS, N2: InValRHS, N3: EltNo);
23245 return DAG.getNode(Opcode: InVecOpcode, DL, VT, N1: LHS, N2: RHS);
23246 }
23247
23248 // Given insert_vector_elt (concat_vectors ...), InVal, Elt
23249 // move the insert_vector_elt to the source operand of the concat_vector.
23250 if (InVec.getOpcode() != ISD::CONCAT_VECTORS)
23251 return SDValue();
23252
23253 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
23254 if (!IndexC)
23255 return SDValue();
23256 unsigned Elt = IndexC->getZExtValue();
23257
23258 EVT ConcatVT = InVec.getOperand(i: 0).getValueType();
23259 if (ConcatVT.getVectorElementType() != InVal.getValueType())
23260 return SDValue();
23261 unsigned ConcatNumElts = ConcatVT.getVectorNumElements();
23262 unsigned NewIdx = Elt % ConcatNumElts;
23263
23264 unsigned ConcatOpIdx = Elt / ConcatNumElts;
23265 SDValue ConcatOp = InVec.getOperand(i: ConcatOpIdx);
23266 ConcatOp = DAG.getInsertVectorElt(DL, Vec: ConcatOp, Elt: InVal, Idx: NewIdx);
23267
23268 SmallVector<SDValue> ConcatOps(InVec->ops());
23269 ConcatOps[ConcatOpIdx] = ConcatOp;
23270 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps);
23271}
23272
23273// If we're concatenating a series of vector loads like
23274// concat_vectors (load v4i8, p+0), (load v4i8, p+n), (load v4i8, p+n*2) ...
23275// Then we can turn this into a strided load by widening the vector elements
23276// vlse32 p, stride=n
23277static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
23278 const RISCVSubtarget &Subtarget,
23279 const RISCVTargetLowering &TLI) {
23280 SDLoc DL(N);
23281 EVT VT = N->getValueType(ResNo: 0);
23282
23283 // Only perform this combine on legal MVTs.
23284 if (!TLI.isTypeLegal(VT))
23285 return SDValue();
23286
23287 // TODO: Potentially extend this to scalable vectors
23288 if (VT.isScalableVector())
23289 return SDValue();
23290
23291 auto *BaseLd = dyn_cast<LoadSDNode>(Val: N->getOperand(Num: 0));
23292 if (!BaseLd || !BaseLd->isSimple() || !ISD::isNormalLoad(N: BaseLd) ||
23293 !SDValue(BaseLd, 0).hasOneUse())
23294 return SDValue();
23295
23296 EVT BaseLdVT = BaseLd->getValueType(ResNo: 0);
23297
23298 // Go through the loads and check that they're strided
23299 SmallVector<LoadSDNode *> Lds;
23300 Lds.push_back(Elt: BaseLd);
23301 Align Align = BaseLd->getAlign();
23302 for (SDValue Op : N->ops().drop_front()) {
23303 auto *Ld = dyn_cast<LoadSDNode>(Val&: Op);
23304 if (!Ld || !Ld->isSimple() || !Op.hasOneUse() ||
23305 Ld->getChain() != BaseLd->getChain() || !ISD::isNormalLoad(N: Ld) ||
23306 Ld->getValueType(ResNo: 0) != BaseLdVT)
23307 return SDValue();
23308
23309 Lds.push_back(Elt: Ld);
23310
23311 // The common alignment is the most restrictive (smallest) of all the loads
23312 Align = std::min(a: Align, b: Ld->getAlign());
23313 }
23314
23315 using PtrDiff = std::pair<std::variant<int64_t, SDValue>, bool>;
23316 auto GetPtrDiff = [&DAG](LoadSDNode *Ld1,
23317 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
23318 // If the load ptrs can be decomposed into a common (Base + Index) with a
23319 // common constant stride, then return the constant stride.
23320 BaseIndexOffset BIO1 = BaseIndexOffset::match(N: Ld1, DAG);
23321 BaseIndexOffset BIO2 = BaseIndexOffset::match(N: Ld2, DAG);
23322 int64_t PtrDiff;
23323 if (BIO1.equalBaseIndex(Other: BIO2, DAG, Off&: PtrDiff))
23324 return {{PtrDiff, false}};
23325
23326 // Otherwise try to match (add LastPtr, Stride) or (add NextPtr, Stride)
23327 SDValue P1 = Ld1->getBasePtr();
23328 SDValue P2 = Ld2->getBasePtr();
23329 if (P2.getOpcode() == ISD::ADD && P2.getOperand(i: 0) == P1)
23330 return {{P2.getOperand(i: 1), false}};
23331 if (P1.getOpcode() == ISD::ADD && P1.getOperand(i: 0) == P2)
23332 return {{P1.getOperand(i: 1), true}};
23333
23334 return std::nullopt;
23335 };
23336
23337 // Get the distance between the first and second loads
23338 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
23339 if (!BaseDiff)
23340 return SDValue();
23341
23342 // Check all the loads are the same distance apart
23343 for (auto *It = Lds.begin() + 1; It != Lds.end() - 1; It++)
23344 if (GetPtrDiff(*It, *std::next(x: It)) != BaseDiff)
23345 return SDValue();
23346
23347 // TODO: At this point, we've successfully matched a generalized gather
23348 // load. Maybe we should emit that, and then move the specialized
23349 // matchers above and below into a DAG combine?
23350
23351 // Get the widened scalar type, e.g. v4i8 -> i64
23352 unsigned WideScalarBitWidth =
23353 BaseLdVT.getScalarSizeInBits() * BaseLdVT.getVectorNumElements();
23354 MVT WideScalarVT = MVT::getIntegerVT(BitWidth: WideScalarBitWidth);
23355
23356 // Get the vector type for the strided load, e.g. 4 x v4i8 -> v4i64
23357 MVT WideVecVT = MVT::getVectorVT(VT: WideScalarVT, NumElements: N->getNumOperands());
23358 if (!TLI.isTypeLegal(VT: WideVecVT))
23359 return SDValue();
23360
23361 // Check that the operation is legal
23362 if (!TLI.isLegalStridedLoadStore(DataType: WideVecVT, Alignment: Align))
23363 return SDValue();
23364
23365 auto [StrideVariant, MustNegateStride] = *BaseDiff;
23366 SDValue Stride =
23367 std::holds_alternative<SDValue>(v: StrideVariant)
23368 ? std::get<SDValue>(v&: StrideVariant)
23369 : DAG.getSignedConstant(Val: std::get<int64_t>(v&: StrideVariant), DL,
23370 VT: Lds[0]->getOffset().getValueType());
23371 if (MustNegateStride)
23372 Stride = DAG.getNegative(Val: Stride, DL, VT: Stride.getValueType());
23373
23374 SDValue AllOneMask =
23375 DAG.getSplat(VT: WideVecVT.changeVectorElementType(EltVT: MVT::i1), DL,
23376 Op: DAG.getConstant(Val: 1, DL, VT: MVT::i1));
23377
23378 uint64_t MemSize;
23379 if (auto *ConstStride = dyn_cast<ConstantSDNode>(Val&: Stride);
23380 ConstStride && ConstStride->getSExtValue() >= 0)
23381 // total size = (elsize * n) + (stride - elsize) * (n-1)
23382 // = elsize + stride * (n-1)
23383 MemSize = WideScalarVT.getSizeInBits() +
23384 ConstStride->getSExtValue() * (N->getNumOperands() - 1);
23385 else
23386 // If Stride isn't constant, then we can't know how much it will load
23387 MemSize = MemoryLocation::UnknownSize;
23388
23389 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
23390 PtrInfo: BaseLd->getPointerInfo(), F: BaseLd->getMemOperand()->getFlags(), Size: MemSize,
23391 BaseAlignment: Align);
23392
23393 SDValue StridedLoad = DAG.getStridedLoadVP(
23394 VT: WideVecVT, DL, Chain: BaseLd->getChain(), Ptr: BaseLd->getBasePtr(), Stride,
23395 Mask: AllOneMask,
23396 EVL: DAG.getConstant(Val: N->getNumOperands(), DL, VT: Subtarget.getXLenVT()), MMO);
23397
23398 for (SDValue Ld : N->ops())
23399 DAG.makeEquivalentMemoryOrdering(OldLoad: cast<LoadSDNode>(Val&: Ld), NewMemOp: StridedLoad);
23400
23401 return DAG.getBitcast(VT: VT.getSimpleVT(), V: StridedLoad);
23402}
23403
23404static SDValue performVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG,
23405 const RISCVSubtarget &Subtarget,
23406 const RISCVTargetLowering &TLI) {
23407 SDLoc DL(N);
23408 EVT VT = N->getValueType(ResNo: 0);
23409 const unsigned ElementSize = VT.getScalarSizeInBits();
23410 const unsigned NumElts = VT.getVectorNumElements();
23411 SDValue V1 = N->getOperand(Num: 0);
23412 SDValue V2 = N->getOperand(Num: 1);
23413 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
23414 ArrayRef<int> Mask = SVN->getMask();
23415 MVT XLenVT = Subtarget.getXLenVT();
23416
23417 // Recognized a disguised select of add/sub.
23418 bool SwapCC;
23419 if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts: NumElts) &&
23420 matchSelectAddSub(TrueVal: V1, FalseVal: V2, SwapCC)) {
23421 SDValue Sub = SwapCC ? V1 : V2;
23422 SDValue A = Sub.getOperand(i: 0);
23423 SDValue B = Sub.getOperand(i: 1);
23424
23425 SmallVector<SDValue> MaskVals;
23426 for (int MaskIndex : Mask) {
23427 bool SelectMaskVal = (MaskIndex < (int)NumElts);
23428 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
23429 }
23430 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
23431 EVT MaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: NumElts);
23432 SDValue CC = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
23433
23434 // Arrange the select such that we can match a masked
23435 // vrsub.vi to perform the conditional negate
23436 SDValue NegB = DAG.getNegative(Val: B, DL, VT);
23437 if (!SwapCC)
23438 CC = DAG.getLogicalNOT(DL, Val: CC, VT: CC->getValueType(ResNo: 0));
23439 SDValue NewB = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CC, N2: NegB, N3: B);
23440 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: NewB);
23441 }
23442
23443 if (SDValue V = compressShuffleOfShuffles(SVN, Subtarget, DAG))
23444 return V;
23445
23446 // Custom legalize <N x i128> or <N x i256> to <M x ELEN>. This runs
23447 // during the combine phase before type legalization, and relies on
23448 // DAGCombine not undoing the transform if isShuffleMaskLegal returns false
23449 // for the source mask.
23450 if (TLI.isTypeLegal(VT) || ElementSize <= Subtarget.getELen() ||
23451 !isPowerOf2_64(Value: ElementSize) || VT.getVectorNumElements() % 2 != 0 ||
23452 VT.isFloatingPoint() || TLI.isShuffleMaskLegal(M: Mask, VT))
23453 return SDValue();
23454
23455 SmallVector<int, 8> NewMask;
23456 narrowShuffleMaskElts(Scale: 2, Mask, ScaledMask&: NewMask);
23457
23458 LLVMContext &C = *DAG.getContext();
23459 EVT NewEltVT = EVT::getIntegerVT(Context&: C, BitWidth: ElementSize / 2);
23460 EVT NewVT = EVT::getVectorVT(Context&: C, VT: NewEltVT, NumElements: VT.getVectorNumElements() * 2);
23461 SDValue Res = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: DAG.getBitcast(VT: NewVT, V: V1),
23462 N2: DAG.getBitcast(VT: NewVT, V: V2), Mask: NewMask);
23463 return DAG.getBitcast(VT, V: Res);
23464}
23465
23466static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG,
23467 const RISCVSubtarget &Subtarget) {
23468 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
23469
23470 if (N->getValueType(ResNo: 0).isFixedLengthVector())
23471 return SDValue();
23472
23473 SDValue Addend = N->getOperand(Num: 0);
23474 SDValue MulOp = N->getOperand(Num: 1);
23475
23476 if (N->getOpcode() == RISCVISD::ADD_VL) {
23477 SDValue AddPassthruOp = N->getOperand(Num: 2);
23478 if (!AddPassthruOp.isUndef())
23479 return SDValue();
23480 }
23481
23482 auto IsVWMulOpc = [](unsigned Opc) {
23483 switch (Opc) {
23484 case RISCVISD::VWMUL_VL:
23485 case RISCVISD::VWMULU_VL:
23486 case RISCVISD::VWMULSU_VL:
23487 return true;
23488 default:
23489 return false;
23490 }
23491 };
23492
23493 if (!IsVWMulOpc(MulOp.getOpcode()))
23494 std::swap(a&: Addend, b&: MulOp);
23495
23496 if (!IsVWMulOpc(MulOp.getOpcode()))
23497 return SDValue();
23498
23499 SDValue MulPassthruOp = MulOp.getOperand(i: 2);
23500
23501 if (!MulPassthruOp.isUndef())
23502 return SDValue();
23503
23504 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
23505 const RISCVSubtarget &Subtarget) {
23506 if (N->getOpcode() == ISD::ADD) {
23507 SDLoc DL(N);
23508 return getDefaultScalableVLOps(VecVT: N->getSimpleValueType(ResNo: 0), DL, DAG,
23509 Subtarget);
23510 }
23511 return std::make_pair(x: N->getOperand(Num: 3), y: N->getOperand(Num: 4));
23512 }(N, DAG, Subtarget);
23513
23514 SDValue MulMask = MulOp.getOperand(i: 3);
23515 SDValue MulVL = MulOp.getOperand(i: 4);
23516
23517 if (AddMask != MulMask || AddVL != MulVL)
23518 return SDValue();
23519
23520 const auto &TSInfo =
23521 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
23522 unsigned Opc = TSInfo.getMAccOpcode(MulOpcode: MulOp.getOpcode());
23523
23524 SDLoc DL(N);
23525 EVT VT = N->getValueType(ResNo: 0);
23526 SDValue Ops[] = {MulOp.getOperand(i: 0), MulOp.getOperand(i: 1), Addend, AddMask,
23527 AddVL};
23528 return DAG.getNode(Opcode: Opc, DL, VT, Ops);
23529}
23530
23531static SDValue combineVdot4aAccum(SDNode *N, SelectionDAG &DAG,
23532 const RISCVSubtarget &Subtarget) {
23533
23534 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
23535
23536 if (!N->getValueType(ResNo: 0).isVector())
23537 return SDValue();
23538
23539 SDValue Addend = N->getOperand(Num: 0);
23540 SDValue DotOp = N->getOperand(Num: 1);
23541
23542 if (N->getOpcode() == RISCVISD::ADD_VL) {
23543 SDValue AddPassthruOp = N->getOperand(Num: 2);
23544 if (!AddPassthruOp.isUndef())
23545 return SDValue();
23546 }
23547
23548 auto IsVdot4aOpc = [](unsigned Opc) {
23549 switch (Opc) {
23550 case RISCVISD::VDOT4A_VL:
23551 case RISCVISD::VDOT4AU_VL:
23552 case RISCVISD::VDOT4ASU_VL:
23553 return true;
23554 default:
23555 return false;
23556 }
23557 };
23558
23559 if (!IsVdot4aOpc(DotOp.getOpcode()))
23560 std::swap(a&: Addend, b&: DotOp);
23561
23562 if (!IsVdot4aOpc(DotOp.getOpcode()))
23563 return SDValue();
23564
23565 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
23566 const RISCVSubtarget &Subtarget) {
23567 if (N->getOpcode() == ISD::ADD) {
23568 SDLoc DL(N);
23569 return getDefaultScalableVLOps(VecVT: N->getSimpleValueType(ResNo: 0), DL, DAG,
23570 Subtarget);
23571 }
23572 return std::make_pair(x: N->getOperand(Num: 3), y: N->getOperand(Num: 4));
23573 }(N, DAG, Subtarget);
23574
23575 SDValue MulVL = DotOp.getOperand(i: 4);
23576 if (AddVL != MulVL)
23577 return SDValue();
23578
23579 if (AddMask.getOpcode() != RISCVISD::VMSET_VL ||
23580 AddMask.getOperand(i: 0) != MulVL)
23581 return SDValue();
23582
23583 SDValue AccumOp = DotOp.getOperand(i: 2);
23584 SDLoc DL(N);
23585 EVT VT = N->getValueType(ResNo: 0);
23586 Addend = DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT, N1: Addend, N2: AccumOp,
23587 N3: DAG.getUNDEF(VT), N4: AddMask, N5: AddVL);
23588
23589 SDValue Ops[] = {DotOp.getOperand(i: 0), DotOp.getOperand(i: 1), Addend,
23590 DotOp.getOperand(i: 3), DotOp->getOperand(Num: 4)};
23591 return DAG.getNode(Opcode: DotOp->getOpcode(), DL, VT, Ops);
23592}
23593
23594static bool
23595legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index,
23596 ISD::MemIndexType &IndexType,
23597 RISCVTargetLowering::DAGCombinerInfo &DCI) {
23598 if (!DCI.isBeforeLegalize())
23599 return false;
23600
23601 SelectionDAG &DAG = DCI.DAG;
23602 const MVT XLenVT =
23603 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>().getXLenVT();
23604
23605 const EVT IndexVT = Index.getValueType();
23606
23607 // RISC-V indexed loads only support the "unsigned unscaled" addressing
23608 // mode, so anything else must be manually legalized.
23609 if (!isIndexTypeSigned(IndexType))
23610 return false;
23611
23612 if (IndexVT.getVectorElementType().bitsLT(VT: XLenVT)) {
23613 // Any index legalization should first promote to XLenVT, so we don't lose
23614 // bits when scaling. This may create an illegal index type so we let
23615 // LLVM's legalization take care of the splitting.
23616 // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
23617 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL,
23618 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: XLenVT,
23619 EC: IndexVT.getVectorElementCount()),
23620 Operand: Index);
23621 }
23622 IndexType = ISD::UNSIGNED_SCALED;
23623 return true;
23624}
23625
23626/// Match the index vector of a scatter or gather node as the shuffle mask
23627/// which performs the rearrangement if possible. Will only match if
23628/// all lanes are touched, and thus replacing the scatter or gather with
23629/// a unit strided access and shuffle is legal.
23630static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask,
23631 SmallVector<int> &ShuffleMask) {
23632 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
23633 return false;
23634 if (!ISD::isBuildVectorOfConstantSDNodes(N: Index.getNode()))
23635 return false;
23636
23637 const unsigned ElementSize = VT.getScalarStoreSize();
23638 const unsigned NumElems = VT.getVectorNumElements();
23639
23640 // Create the shuffle mask and check all bits active
23641 assert(ShuffleMask.empty());
23642 BitVector ActiveLanes(NumElems);
23643 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
23644 // TODO: We've found an active bit of UB, and could be
23645 // more aggressive here if desired.
23646 if (Index->getOperand(Num: i)->isUndef())
23647 return false;
23648 uint64_t C = Index->getConstantOperandVal(Num: i);
23649 if (C % ElementSize != 0)
23650 return false;
23651 C = C / ElementSize;
23652 if (C >= NumElems)
23653 return false;
23654 ShuffleMask.push_back(Elt: C);
23655 ActiveLanes.set(C);
23656 }
23657 return ActiveLanes.all();
23658}
23659
23660/// Match the index of a gather or scatter operation as an operation
23661/// with twice the element width and half the number of elements. This is
23662/// generally profitable (if legal) because these operations are linear
23663/// in VL, so even if we cause some extract VTYPE/VL toggles, we still
23664/// come out ahead.
23665static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask,
23666 Align BaseAlign, const RISCVSubtarget &ST) {
23667 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
23668 return false;
23669 if (!ISD::isBuildVectorOfConstantSDNodes(N: Index.getNode()))
23670 return false;
23671
23672 // Attempt a doubling. If we can use a element type 4x or 8x in
23673 // size, this will happen via multiply iterations of the transform.
23674 const unsigned NumElems = VT.getVectorNumElements();
23675 if (NumElems % 2 != 0)
23676 return false;
23677
23678 const unsigned ElementSize = VT.getScalarStoreSize();
23679 const unsigned WiderElementSize = ElementSize * 2;
23680 if (WiderElementSize > ST.getELen()/8)
23681 return false;
23682
23683 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
23684 return false;
23685
23686 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
23687 // TODO: We've found an active bit of UB, and could be
23688 // more aggressive here if desired.
23689 if (Index->getOperand(Num: i)->isUndef())
23690 return false;
23691 // TODO: This offset check is too strict if we support fully
23692 // misaligned memory operations.
23693 uint64_t C = Index->getConstantOperandVal(Num: i);
23694 if (i % 2 == 0) {
23695 if (C % WiderElementSize != 0)
23696 return false;
23697 continue;
23698 }
23699 uint64_t Last = Index->getConstantOperandVal(Num: i-1);
23700 if (C != Last + ElementSize)
23701 return false;
23702 }
23703 return true;
23704}
23705
23706// trunc (sra sext (X), zext (Y)) -> sra (X, smin (Y, scalarsize(Y) - 1))
23707// This would be benefit for the cases where X and Y are both the same value
23708// type of low precision vectors. Since the truncate would be lowered into
23709// n-levels TRUNCATE_VECTOR_VL to satisfy RVV's SEW*2->SEW truncate
23710// restriction, such pattern would be expanded into a series of "vsetvli"
23711// and "vnsrl" instructions later to reach this point.
23712static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG) {
23713 SDValue Mask = N->getOperand(Num: 1);
23714 SDValue VL = N->getOperand(Num: 2);
23715
23716 bool IsVLMAX = isAllOnesConstant(V: VL) ||
23717 (isa<RegisterSDNode>(Val: VL) &&
23718 cast<RegisterSDNode>(Val&: VL)->getReg() == RISCV::X0);
23719 if (!IsVLMAX || Mask.getOpcode() != RISCVISD::VMSET_VL ||
23720 Mask.getOperand(i: 0) != VL)
23721 return SDValue();
23722
23723 auto IsTruncNode = [&](SDValue V) {
23724 return V.getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL &&
23725 V.getOperand(i: 1) == Mask && V.getOperand(i: 2) == VL;
23726 };
23727
23728 SDValue Op = N->getOperand(Num: 0);
23729
23730 // We need to first find the inner level of TRUNCATE_VECTOR_VL node
23731 // to distinguish such pattern.
23732 while (IsTruncNode(Op)) {
23733 if (!Op.hasOneUse())
23734 return SDValue();
23735 Op = Op.getOperand(i: 0);
23736 }
23737
23738 if (Op.getOpcode() != ISD::SRA || !Op.hasOneUse())
23739 return SDValue();
23740
23741 SDValue N0 = Op.getOperand(i: 0);
23742 SDValue N1 = Op.getOperand(i: 1);
23743 if (N0.getOpcode() != ISD::SIGN_EXTEND || !N0.hasOneUse() ||
23744 N1.getOpcode() != ISD::ZERO_EXTEND || !N1.hasOneUse())
23745 return SDValue();
23746
23747 SDValue N00 = N0.getOperand(i: 0);
23748 SDValue N10 = N1.getOperand(i: 0);
23749 if (!N00.getValueType().isVector() ||
23750 N00.getValueType() != N10.getValueType() ||
23751 N->getValueType(ResNo: 0) != N10.getValueType())
23752 return SDValue();
23753
23754 unsigned MaxShAmt = N10.getValueType().getScalarSizeInBits() - 1;
23755 SDValue SMin =
23756 DAG.getNode(Opcode: ISD::SMIN, DL: SDLoc(N1), VT: N->getValueType(ResNo: 0), N1: N10,
23757 N2: DAG.getConstant(Val: MaxShAmt, DL: SDLoc(N1), VT: N->getValueType(ResNo: 0)));
23758 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: N00, N2: SMin);
23759}
23760
23761// Convert
23762// (iX ctpop (bitcast (vXi1 A)))
23763// ->
23764// (zext (vcpop.m (nxvYi1 (insert_subvec (vXi1 A)))))
23765// and
23766// (iN reduce.add (zext (vXi1 A to vXiN))
23767// ->
23768// (zext (vcpop.m (nxvYi1 (insert_subvec (vXi1 A)))))
23769// FIXME: It's complicated to match all the variations of this after type
23770// legalization so we only handle the pre-type legalization pattern, but that
23771// requires the fixed vector type to be legal.
23772static SDValue combineToVCPOP(SDNode *N, SelectionDAG &DAG,
23773 const RISCVSubtarget &Subtarget) {
23774 unsigned Opc = N->getOpcode();
23775 assert((Opc == ISD::CTPOP || Opc == ISD::VECREDUCE_ADD) &&
23776 "Unexpected opcode");
23777 EVT VT = N->getValueType(ResNo: 0);
23778 if (!VT.isScalarInteger())
23779 return SDValue();
23780
23781 SDValue Src = N->getOperand(Num: 0);
23782
23783 if (Opc == ISD::CTPOP) {
23784 // Peek through zero_extend. It doesn't change the count.
23785 if (Src.getOpcode() == ISD::ZERO_EXTEND)
23786 Src = Src.getOperand(i: 0);
23787
23788 if (Src.getOpcode() != ISD::BITCAST)
23789 return SDValue();
23790 Src = Src.getOperand(i: 0);
23791 } else if (Opc == ISD::VECREDUCE_ADD) {
23792 if (Src.getOpcode() != ISD::ZERO_EXTEND)
23793 return SDValue();
23794 Src = Src.getOperand(i: 0);
23795 }
23796
23797 EVT SrcEVT = Src.getValueType();
23798 if (!SrcEVT.isSimple())
23799 return SDValue();
23800
23801 MVT SrcMVT = SrcEVT.getSimpleVT();
23802 // Make sure the input is an i1 vector.
23803 if (!SrcMVT.isVectorOf(EltVT: MVT::i1))
23804 return SDValue();
23805
23806 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23807 if (!TLI.isTypeLegal(VT: SrcMVT))
23808 return SDValue();
23809
23810 // Check that destination type is large enough to hold result without
23811 // overflow.
23812 if (Opc == ISD::VECREDUCE_ADD) {
23813 unsigned EltSize = SrcMVT.getScalarSizeInBits();
23814 unsigned MinSize = SrcMVT.getSizeInBits().getKnownMinValue();
23815 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
23816 unsigned MaxVLMAX = SrcMVT.isFixedLengthVector()
23817 ? SrcMVT.getVectorNumElements()
23818 : RISCVTargetLowering::computeVLMAX(
23819 VectorBits: VectorBitsMax, EltSize, MinSize);
23820 if (VT.getFixedSizeInBits() < Log2_32(Value: MaxVLMAX) + 1)
23821 return SDValue();
23822 }
23823
23824 MVT ContainerVT = SrcMVT;
23825 if (SrcMVT.isFixedLengthVector()) {
23826 ContainerVT = getContainerForFixedLengthVector(VT: SrcMVT, Subtarget);
23827 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
23828 }
23829
23830 SDLoc DL(N);
23831 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcMVT, ContainerVT, DL, DAG, Subtarget);
23832
23833 MVT XLenVT = Subtarget.getXLenVT();
23834 SDValue Pop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Src, N2: Mask, N3: VL);
23835 return DAG.getZExtOrTrunc(Op: Pop, DL, VT);
23836}
23837
23838static SDValue performSHLCombine(SDNode *N,
23839 TargetLowering::DAGCombinerInfo &DCI,
23840 const RISCVSubtarget &Subtarget) {
23841 // (shl (zext x), y) -> (vwsll x, y)
23842 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
23843 return V;
23844
23845 // (shl (sext x), C) -> (vwmulsu x, 1u << C)
23846 // (shl (zext x), C) -> (vwmulu x, 1u << C)
23847
23848 if (!DCI.isAfterLegalizeDAG())
23849 return SDValue();
23850
23851 SDValue LHS = N->getOperand(Num: 0);
23852 if (!LHS.hasOneUse())
23853 return SDValue();
23854 unsigned Opcode;
23855 switch (LHS.getOpcode()) {
23856 case ISD::SIGN_EXTEND:
23857 case RISCVISD::VSEXT_VL:
23858 Opcode = RISCVISD::VWMULSU_VL;
23859 break;
23860 case ISD::ZERO_EXTEND:
23861 case RISCVISD::VZEXT_VL:
23862 Opcode = RISCVISD::VWMULU_VL;
23863 break;
23864 default:
23865 return SDValue();
23866 }
23867
23868 SDValue RHS = N->getOperand(Num: 1);
23869 APInt ShAmt;
23870 uint64_t ShAmtInt;
23871 if (ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: ShAmt))
23872 ShAmtInt = ShAmt.getZExtValue();
23873 else if (RHS.getOpcode() == RISCVISD::VMV_V_X_VL &&
23874 RHS.getOperand(i: 1).getOpcode() == ISD::Constant)
23875 ShAmtInt = RHS.getConstantOperandVal(i: 1);
23876 else
23877 return SDValue();
23878
23879 // Better foldings:
23880 // (shl (sext x), 1) -> (vwadd x, x)
23881 // (shl (zext x), 1) -> (vwaddu x, x)
23882 if (ShAmtInt <= 1)
23883 return SDValue();
23884
23885 SDValue NarrowOp = LHS.getOperand(i: 0);
23886 MVT NarrowVT = NarrowOp.getSimpleValueType();
23887 uint64_t NarrowBits = NarrowVT.getScalarSizeInBits();
23888 if (ShAmtInt >= NarrowBits)
23889 return SDValue();
23890 MVT VT = N->getSimpleValueType(ResNo: 0);
23891 if (NarrowBits * 2 != VT.getScalarSizeInBits())
23892 return SDValue();
23893
23894 SelectionDAG &DAG = DCI.DAG;
23895 SDLoc DL(N);
23896 SDValue Passthru, Mask, VL;
23897 switch (N->getOpcode()) {
23898 case ISD::SHL:
23899 Passthru = DAG.getUNDEF(VT);
23900 std::tie(args&: Mask, args&: VL) = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
23901 break;
23902 case RISCVISD::SHL_VL:
23903 Passthru = N->getOperand(Num: 2);
23904 Mask = N->getOperand(Num: 3);
23905 VL = N->getOperand(Num: 4);
23906 break;
23907 default:
23908 llvm_unreachable("Expected SHL");
23909 }
23910 return DAG.getNode(Opcode, DL, VT, N1: NarrowOp,
23911 N2: DAG.getConstant(Val: 1ULL << ShAmtInt, DL: SDLoc(RHS), VT: NarrowVT),
23912 N3: Passthru, N4: Mask, N5: VL);
23913}
23914
23915// Fold (smax (smin X, (1 << C) - 1), -(1 << C)) -> riscv_sati X, C.
23916// Fold (smin (smax X, -(1 << C)), (1 << C) - 1) -> riscv_sati X, C.
23917// Fold (smax (smin X, (1 << C) - 1), 0) -> riscv_usati X, C.
23918// Fold (smin (smax X, 0, (1 << C) - 1) -> riscv_usati X, C.
23919static SDValue combineMinMaxToSat(SDNode *N,
23920 TargetLowering::DAGCombinerInfo &DCI,
23921 const RISCVSubtarget &Subtarget) {
23922 if (!DCI.isAfterLegalizeDAG())
23923 return SDValue();
23924
23925 if (!Subtarget.hasStdExtP())
23926 return SDValue();
23927
23928 EVT VT = N->getValueType(ResNo: 0);
23929
23930 if (VT != Subtarget.getXLenVT())
23931 return SDValue();
23932
23933 SDValue N0 = N->getOperand(Num: 0);
23934
23935 if ((N0.getOpcode() != ISD::SMIN && N0.getOpcode() != ISD::SMAX) ||
23936 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) ||
23937 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
23938 return SDValue();
23939
23940 SDValue Min = SDValue(N, 0);
23941 SDValue Max = N0;
23942 SDValue Input = N0.getOperand(i: 0);
23943 if (Min.getOpcode() == ISD::SMAX)
23944 std::swap(a&: Min, b&: Max);
23945
23946 APInt MinC = Min.getConstantOperandAPInt(i: 1);
23947 APInt MaxC = Max.getConstantOperandAPInt(i: 1);
23948
23949 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX ||
23950 !(MinC + 1).isPowerOf2())
23951 return SDValue();
23952
23953 SelectionDAG &DAG = DCI.DAG;
23954
23955 SDLoc DL(N);
23956 if (MinC == ~MaxC)
23957 return DAG.getNode(Opcode: RISCVISD::SATI, DL, VT, N1: Input,
23958 N2: DAG.getTargetConstant(Val: MinC.countr_one(), DL, VT));
23959 if (MaxC == 0)
23960 return DAG.getNode(Opcode: RISCVISD::USATI, DL, VT, N1: Input,
23961 N2: DAG.getTargetConstant(Val: MinC.countr_one(), DL, VT));
23962
23963 return SDValue();
23964}
23965
23966// Returns true if the i32 pair (Lo, Hi) is the 64-bit sign-extension of the
23967// i32 value Lo, i.e. Hi == (sra Lo, 31). Used to fold ADDD/SUBD of a
23968// sign-extended operand into the WADDA/WSUBA widening accumulate nodes.
23969static bool isI32SignExtended(SDValue Lo, SDValue Hi) {
23970 return Hi.getOpcode() == ISD::SRA && Hi.getOperand(i: 0) == Lo &&
23971 isa<ConstantSDNode>(Val: Hi.getOperand(i: 1)) &&
23972 Hi.getConstantOperandVal(i: 1) == 31;
23973}
23974
23975SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
23976 DAGCombinerInfo &DCI) const {
23977 SelectionDAG &DAG = DCI.DAG;
23978 const MVT XLenVT = Subtarget.getXLenVT();
23979 SDLoc DL(N);
23980
23981 // Helper to call SimplifyDemandedBits on an operand of N where only some low
23982 // bits are demanded. N will be added to the Worklist if it was not deleted.
23983 // Caller should return SDValue(N, 0) if this returns true.
23984 auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
23985 SDValue Op = N->getOperand(Num: OpNo);
23986 APInt Mask = APInt::getLowBitsSet(numBits: Op.getValueSizeInBits(), loBitsSet: LowBits);
23987 if (!SimplifyDemandedBits(Op, DemandedBits: Mask, DCI))
23988 return false;
23989
23990 if (N->getOpcode() != ISD::DELETED_NODE)
23991 DCI.AddToWorklist(N);
23992 return true;
23993 };
23994
23995 switch (N->getOpcode()) {
23996 default:
23997 break;
23998 case RISCVISD::SplitF64: {
23999 SDValue Op0 = N->getOperand(Num: 0);
24000 // If the input to SplitF64 is just BuildPairF64 then the operation is
24001 // redundant. Instead, use BuildPairF64's operands directly.
24002 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
24003 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
24004
24005 if (Op0->isUndef()) {
24006 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
24007 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
24008 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
24009 }
24010
24011 // It's cheaper to materialise two 32-bit integers than to load a double
24012 // from the constant pool and transfer it to integer registers through the
24013 // stack.
24014 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
24015 APInt V = C->getValueAPF().bitcastToAPInt();
24016 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
24017 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
24018 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
24019 }
24020
24021 // This is a target-specific version of a DAGCombine performed in
24022 // DAGCombiner::visitBITCAST. It performs the equivalent of:
24023 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
24024 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
24025 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
24026 !Op0.getNode()->hasOneUse() || Subtarget.hasStdExtZdinx())
24027 break;
24028 SDValue NewSplitF64 =
24029 DAG.getNode(Opcode: RISCVISD::SplitF64, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24030 N: Op0.getOperand(i: 0));
24031 SDValue Lo = NewSplitF64.getValue(R: 0);
24032 SDValue Hi = NewSplitF64.getValue(R: 1);
24033 APInt SignBit = APInt::getSignMask(BitWidth: 32);
24034 if (Op0.getOpcode() == ISD::FNEG) {
24035 SDValue NewHi = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Hi,
24036 N2: DAG.getConstant(Val: SignBit, DL, VT: MVT::i32));
24037 return DCI.CombineTo(N, Res0: Lo, Res1: NewHi);
24038 }
24039 assert(Op0.getOpcode() == ISD::FABS);
24040 SDValue NewHi = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Hi,
24041 N2: DAG.getConstant(Val: ~SignBit, DL, VT: MVT::i32));
24042 return DCI.CombineTo(N, Res0: Lo, Res1: NewHi);
24043 }
24044 case RISCVISD::SLLW:
24045 case RISCVISD::SRAW:
24046 case RISCVISD::SRLW:
24047 case RISCVISD::RORW:
24048 case RISCVISD::ROLW: {
24049 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
24050 if (SimplifyDemandedLowBitsHelper(0, 32) ||
24051 SimplifyDemandedLowBitsHelper(1, 5))
24052 return SDValue(N, 0);
24053
24054 break;
24055 }
24056 case RISCVISD::ABSW:
24057 case RISCVISD::CLSW:
24058 case RISCVISD::CLZW:
24059 case RISCVISD::CTZW: {
24060 // Only the lower 32 bits of the first operand are read
24061 if (SimplifyDemandedLowBitsHelper(0, 32))
24062 return SDValue(N, 0);
24063 break;
24064 }
24065 case RISCVISD::WMULSU: {
24066 // Convert to MULHSU if only the upper half is used.
24067 if (!N->hasAnyUseOfValue(Value: 0)) {
24068 SDValue Res = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: N->getValueType(ResNo: 1),
24069 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
24070 return DCI.CombineTo(N, Res0: Res, Res1: Res);
24071 }
24072 break;
24073 }
24074 case RISCVISD::PSRL:
24075 case RISCVISD::PSRA: {
24076 // Fold (PSRL/PSRA (trunc (PSRL X, C1)), C2) -> (trunc (PSRL/PSRA X, C1+C2))
24077 // Fold (PSRL/PSRA (concat (trunc (PSRL X, C1)), (trunc (PSRL Y, C1))), C2)
24078 // -> (concat (trunc (PSRL/PSRA X, C1+C2)), (trunc (PSRL/PSRA Y, C1+C2)))
24079 // In both cases C1 must equal the number of bits discarded by the truncate.
24080 auto *C2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
24081 if (!C2)
24082 break;
24083 // Match (trunc (PSRL X, C1)) where C1 == bits discarded by the truncate
24084 // and C1+C2 is a valid shift. Returns {NarrowVT, WideVT, NewShAmt, X}
24085 // without creating any DAG nodes.
24086 struct TruncPSRLMatch {
24087 uint64_t NewShAmt;
24088 SDValue Src;
24089 };
24090 auto MatchTruncPSRL =
24091 [](SDValue TruncVal,
24092 ConstantSDNode *C2) -> std::optional<TruncPSRLMatch> {
24093 if (TruncVal.getOpcode() != ISD::TRUNCATE || !TruncVal.hasOneUse())
24094 return std::nullopt;
24095 SDValue PSRLVal = TruncVal.getOperand(i: 0);
24096 if (PSRLVal.getOpcode() != RISCVISD::PSRL || !PSRLVal.hasOneUse())
24097 return std::nullopt;
24098 auto *C1 = dyn_cast<ConstantSDNode>(Val: PSRLVal.getOperand(i: 1));
24099 if (!C1)
24100 return std::nullopt;
24101 MVT NarrowVT = TruncVal.getSimpleValueType();
24102 MVT WideVT = PSRLVal.getSimpleValueType();
24103 unsigned WideEltBits = WideVT.getVectorElementType().getSizeInBits();
24104 unsigned NarrowEltBits = NarrowVT.getVectorElementType().getSizeInBits();
24105 if (C1->getZExtValue() != WideEltBits - NarrowEltBits)
24106 return std::nullopt;
24107 uint64_t NewShAmt = C1->getZExtValue() + C2->getZExtValue();
24108 if (NewShAmt >= WideEltBits)
24109 return std::nullopt;
24110 return TruncPSRLMatch{.NewShAmt: NewShAmt, .Src: PSRLVal.getOperand(i: 0)};
24111 };
24112 auto MakeFoldedShift = [&](const TruncPSRLMatch &M, EVT VT,
24113 unsigned OuterOpc) {
24114 SDValue NewShift = DAG.getNode(Opcode: OuterOpc, DL, VT: M.Src.getValueType(), N1: M.Src,
24115 N2: DAG.getConstant(Val: M.NewShAmt, DL, VT: XLenVT));
24116 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewShift);
24117 };
24118
24119 SDValue Src = N->getOperand(Num: 0);
24120 if (auto M = MatchTruncPSRL(Src, C2))
24121 return MakeFoldedShift(*M, Src.getValueType(), N->getOpcode());
24122
24123 if (Src.getOpcode() == ISD::CONCAT_VECTORS && Src.hasOneUse() &&
24124 Src.getNumOperands() == 2) {
24125 SDValue Op0 = Src.getOperand(i: 0);
24126 SDValue Op1 = Src.getOperand(i: 1);
24127 auto M0 = MatchTruncPSRL(Op0, C2);
24128 auto M1 = MatchTruncPSRL(Op1, C2);
24129 if (M0 && M1)
24130 return DAG.getNode(
24131 Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0),
24132 N1: MakeFoldedShift(*M0, Op0.getValueType(), N->getOpcode()),
24133 N2: MakeFoldedShift(*M1, Op1.getValueType(), N->getOpcode()));
24134 }
24135
24136 break;
24137 }
24138 case RISCVISD::ADDD: {
24139 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
24140 "ADDD is only for RV32 with P extension");
24141
24142 SDValue Op0Lo = N->getOperand(Num: 0);
24143 SDValue Op0Hi = N->getOperand(Num: 1);
24144 SDValue Op1Lo = N->getOperand(Num: 2);
24145 SDValue Op1Hi = N->getOperand(Num: 3);
24146
24147 // (ADDD lo, hi, x, 0) -> (WADDAU lo, hi, x, 0)
24148 if (isNullConstant(V: Op1Hi)) {
24149 SDValue Result =
24150 DAG.getNode(Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24151 N1: Op0Lo, N2: Op0Hi, N3: Op1Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
24152 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24153 }
24154 // (ADDD x, 0, lo, hi) -> (WADDAU lo, hi, x, 0)
24155 if (isNullConstant(V: Op0Hi)) {
24156 SDValue Result =
24157 DAG.getNode(Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24158 N1: Op1Lo, N2: Op1Hi, N3: Op0Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
24159 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24160 }
24161
24162 // (ADDD lo, hi, x, sra(x, 31)) -> (WADDA lo, hi, x, 0)
24163 if (isI32SignExtended(Lo: Op1Lo, Hi: Op1Hi)) {
24164 SDValue Result =
24165 DAG.getNode(Opcode: RISCVISD::WADDA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24166 N1: Op0Lo, N2: Op0Hi, N3: Op1Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
24167 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24168 }
24169 // (ADDD x, sra(x, 31), lo, hi) -> (WADDA lo, hi, x, 0)
24170 if (isI32SignExtended(Lo: Op0Lo, Hi: Op0Hi)) {
24171 SDValue Result =
24172 DAG.getNode(Opcode: RISCVISD::WADDA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24173 N1: Op1Lo, N2: Op1Hi, N3: Op0Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
24174 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24175 }
24176 break;
24177 }
24178 case RISCVISD::SUBD: {
24179 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
24180 "SUBD is only for RV32 with P extension");
24181
24182 SDValue Op0Lo = N->getOperand(Num: 0);
24183 SDValue Op0Hi = N->getOperand(Num: 1);
24184 SDValue Op1Lo = N->getOperand(Num: 2);
24185 SDValue Op1Hi = N->getOperand(Num: 3);
24186
24187 // (SUBD lo, hi, x, 0) -> (WSUBAU lo, hi, 0, x)
24188 // WSUBAU semantics: rd = rd + zext(rs1) - zext(rs2)
24189 if (isNullConstant(V: Op1Hi)) {
24190 SDValue Result =
24191 DAG.getNode(Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24192 N1: Op0Lo, N2: Op0Hi, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N4: Op1Lo);
24193 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24194 }
24195
24196 // (SUBD lo, hi, x, sra(x, 31)) -> (WSUBA lo, hi, 0, x)
24197 // WSUBA semantics: rd = rd + sext(rs1) - sext(rs2)
24198 if (isI32SignExtended(Lo: Op1Lo, Hi: Op1Hi)) {
24199 SDValue Result =
24200 DAG.getNode(Opcode: RISCVISD::WSUBA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24201 N1: Op0Lo, N2: Op0Hi, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N4: Op1Lo);
24202 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24203 }
24204 break;
24205 }
24206 case RISCVISD::WADDAU: {
24207 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
24208 "WADDAU is only for RV32 with P extension");
24209 SDValue Op0Lo = N->getOperand(Num: 0);
24210 SDValue Op0Hi = N->getOperand(Num: 1);
24211 SDValue Op1 = N->getOperand(Num: 2);
24212 SDValue Op2 = N->getOperand(Num: 3);
24213
24214 // (WADDAU lo, 0, rs1, 0) -> (WADDU lo, rs1)
24215 if (isNullConstant(V: Op0Hi) && isNullConstant(V: Op2)) {
24216 SDValue Result = DAG.getNode(
24217 Opcode: RISCVISD::WADDU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op1);
24218 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24219 }
24220
24221 // (WADDAU -C, -1, rs1, 0) -> (WSUBU rs1, C) where C > 0
24222 if (isNullConstant(V: Op2) && isAllOnesConstant(V: Op0Hi)) {
24223 if (auto *C0 = dyn_cast<ConstantSDNode>(Val&: Op0Lo)) {
24224 int64_t Val = C0->getSExtValue();
24225 if (Val < 0) {
24226 SDValue PosConst = DAG.getConstant(Val: -Val, DL, VT: MVT::i32);
24227 SDValue Result =
24228 DAG.getNode(Opcode: RISCVISD::WSUBU, DL,
24229 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op1, N2: PosConst);
24230 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24231 }
24232 }
24233 }
24234
24235 // FIXME: Canonicalize zero Op1 to Op2.
24236 if (isNullConstant(V: Op2) && Op0Lo.getNode() == Op0Hi.getNode() &&
24237 Op0Lo.getResNo() == 0 && Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() &&
24238 Op0Hi.hasOneUse()) {
24239 // (WADDAU (WADDAU lo, hi, x, 0), y, 0) -> (WADDAU lo, hi, x, y)
24240 if (Op0Lo.getOpcode() == RISCVISD::WADDAU &&
24241 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
24242 SDValue Result = DAG.getNode(
24243 Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24244 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op1);
24245 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24246 }
24247 // (WADDAU (WSUBAU lo, hi, 0, a), b, 0) -> (WSUBAU lo, hi, b, a)
24248 if (Op0Lo.getOpcode() == RISCVISD::WSUBAU &&
24249 isNullConstant(V: Op0Lo.getOperand(i: 2))) {
24250 SDValue Result = DAG.getNode(
24251 Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24252 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op1, N4: Op0Lo.getOperand(i: 3));
24253 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24254 }
24255 }
24256 break;
24257 }
24258 case RISCVISD::WSUBAU: {
24259 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
24260 "WSUBAU is only for RV32 with P extension");
24261 SDValue Op0Lo = N->getOperand(Num: 0);
24262 SDValue Op0Hi = N->getOperand(Num: 1);
24263 SDValue Op1 = N->getOperand(Num: 2);
24264 SDValue Op2 = N->getOperand(Num: 3);
24265
24266 // (WSUBAU lo, 0, 0, rs2) -> (WSUBU lo, rs2)
24267 if (isNullConstant(V: Op0Hi) && isNullConstant(V: Op1)) {
24268 SDValue Result = DAG.getNode(
24269 Opcode: RISCVISD::WSUBU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op2);
24270 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24271 }
24272
24273 // (WSUBAU (WADDAU lo, hi, a, 0), 0, b) -> (WSUBAU lo, hi, a, b)
24274 if (isNullConstant(V: Op1) && Op0Lo.getOpcode() == RISCVISD::WADDAU &&
24275 Op0Lo.getNode() == Op0Hi.getNode() && Op0Lo.getResNo() == 0 &&
24276 Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() && Op0Hi.hasOneUse() &&
24277 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
24278 SDValue Result = DAG.getNode(
24279 Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24280 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op2);
24281 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24282 }
24283 break;
24284 }
24285 case RISCVISD::WADDA: {
24286 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
24287 "WADDA is only for RV32 with P extension");
24288 SDValue Op0Lo = N->getOperand(Num: 0);
24289 SDValue Op0Hi = N->getOperand(Num: 1);
24290 SDValue Op1 = N->getOperand(Num: 2);
24291 SDValue Op2 = N->getOperand(Num: 3);
24292
24293 // (WADDA lo, sra(lo, 31), rs1, 0) -> (WADD lo, rs1)
24294 if (isNullConstant(V: Op2) && isI32SignExtended(Lo: Op0Lo, Hi: Op0Hi)) {
24295 SDValue Result = DAG.getNode(
24296 Opcode: RISCVISD::WADD, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op1);
24297 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24298 }
24299
24300 // Fold a chained accumulate into the free second source slot.
24301 if (isNullConstant(V: Op2) && Op0Lo.getNode() == Op0Hi.getNode() &&
24302 Op0Lo.getResNo() == 0 && Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() &&
24303 Op0Hi.hasOneUse()) {
24304 // (WADDA (WADDA lo, hi, x, 0), y, 0) -> (WADDA lo, hi, x, y)
24305 if (Op0Lo.getOpcode() == RISCVISD::WADDA &&
24306 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
24307 SDValue Result = DAG.getNode(
24308 Opcode: RISCVISD::WADDA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24309 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op1);
24310 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24311 }
24312 // (WADDA (WSUBA lo, hi, 0, a), b, 0) -> (WSUBA lo, hi, b, a)
24313 if (Op0Lo.getOpcode() == RISCVISD::WSUBA &&
24314 isNullConstant(V: Op0Lo.getOperand(i: 2))) {
24315 SDValue Result = DAG.getNode(
24316 Opcode: RISCVISD::WSUBA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24317 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op1, N4: Op0Lo.getOperand(i: 3));
24318 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24319 }
24320 }
24321 break;
24322 }
24323 case RISCVISD::WSUBA: {
24324 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
24325 "WSUBA is only for RV32 with P extension");
24326 SDValue Op0Lo = N->getOperand(Num: 0);
24327 SDValue Op0Hi = N->getOperand(Num: 1);
24328 SDValue Op1 = N->getOperand(Num: 2);
24329 SDValue Op2 = N->getOperand(Num: 3);
24330
24331 // (WSUBA lo, sra(lo, 31), 0, rs2) -> (WSUB lo, rs2)
24332 if (isNullConstant(V: Op1) && isI32SignExtended(Lo: Op0Lo, Hi: Op0Hi)) {
24333 SDValue Result = DAG.getNode(
24334 Opcode: RISCVISD::WSUB, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op2);
24335 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24336 }
24337
24338 // (WSUBA (WADDA lo, hi, a, 0), 0, b) -> (WSUBA lo, hi, a, b)
24339 if (isNullConstant(V: Op1) && Op0Lo.getOpcode() == RISCVISD::WADDA &&
24340 Op0Lo.getNode() == Op0Hi.getNode() && Op0Lo.getResNo() == 0 &&
24341 Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() && Op0Hi.hasOneUse() &&
24342 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
24343 SDValue Result = DAG.getNode(
24344 Opcode: RISCVISD::WSUBA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
24345 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op2);
24346 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
24347 }
24348 break;
24349 }
24350 case RISCVISD::FMV_W_X_RV64: {
24351 // If the input to FMV_W_X_RV64 is just FMV_X_ANYEXTW_RV64 the the
24352 // conversion is unnecessary and can be replaced with the
24353 // FMV_X_ANYEXTW_RV64 operand.
24354 SDValue Op0 = N->getOperand(Num: 0);
24355 if (Op0.getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64)
24356 return Op0.getOperand(i: 0);
24357 break;
24358 }
24359 case RISCVISD::FMV_X_ANYEXTH:
24360 case RISCVISD::FMV_X_ANYEXTW_RV64: {
24361 SDLoc DL(N);
24362 SDValue Op0 = N->getOperand(Num: 0);
24363 MVT VT = N->getSimpleValueType(ResNo: 0);
24364
24365 // Constant fold.
24366 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
24367 APInt Val = CFP->getValueAPF().bitcastToAPInt().sext(width: VT.getSizeInBits());
24368 return DAG.getConstant(Val, DL, VT);
24369 }
24370
24371 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
24372 // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
24373 // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
24374 if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
24375 Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
24376 (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
24377 Op0->getOpcode() == RISCVISD::FMV_H_X)) {
24378 assert(Op0.getOperand(0).getValueType() == VT &&
24379 "Unexpected value type!");
24380 return Op0.getOperand(i: 0);
24381 }
24382
24383 if (ISD::isNormalLoad(N: Op0.getNode()) && Op0.hasOneUse() &&
24384 cast<LoadSDNode>(Val&: Op0)->isSimple()) {
24385 MVT IVT = MVT::getIntegerVT(BitWidth: Op0.getValueSizeInBits());
24386 auto *LN0 = cast<LoadSDNode>(Val&: Op0);
24387 SDValue Load =
24388 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: SDLoc(N), VT, Chain: LN0->getChain(),
24389 Ptr: LN0->getBasePtr(), MemVT: IVT, MMO: LN0->getMemOperand());
24390 DAG.ReplaceAllUsesOfValueWith(From: Op0.getValue(R: 1), To: Load.getValue(R: 1));
24391 return Load;
24392 }
24393
24394 // This is a target-specific version of a DAGCombine performed in
24395 // DAGCombiner::visitBITCAST. It performs the equivalent of:
24396 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
24397 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
24398 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
24399 !Op0.getNode()->hasOneUse())
24400 break;
24401 SDValue NewFMV = DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Op0.getOperand(i: 0));
24402 unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
24403 APInt SignBit = APInt::getSignMask(BitWidth: FPBits).sext(width: VT.getSizeInBits());
24404 if (Op0.getOpcode() == ISD::FNEG)
24405 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewFMV,
24406 N2: DAG.getConstant(Val: SignBit, DL, VT));
24407
24408 assert(Op0.getOpcode() == ISD::FABS);
24409 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NewFMV,
24410 N2: DAG.getConstant(Val: ~SignBit, DL, VT));
24411 }
24412 case ISD::ABS:
24413 case ISD::ABS_MIN_POISON: {
24414 EVT VT = N->getValueType(ResNo: 0);
24415 SDValue N0 = N->getOperand(Num: 0);
24416 // abs (sext) -> zext (abs)
24417 // abs (zext) -> zext (handled elsewhere)
24418 if (VT.isVector() && N0.hasOneUse() && N0.getOpcode() == ISD::SIGN_EXTEND) {
24419 SDValue Src = N0.getOperand(i: 0);
24420 SDLoc DL(N);
24421 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT,
24422 Operand: DAG.getNode(Opcode: ISD::ABS, DL, VT: Src.getValueType(), Operand: Src));
24423 }
24424 break;
24425 }
24426 case ISD::ADD: {
24427 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
24428 return V;
24429 if (SDValue V = combineToVWMACC(N, DAG, Subtarget))
24430 return V;
24431 if (SDValue V = combineVdot4aAccum(N, DAG, Subtarget))
24432 return V;
24433 return performADDCombine(N, DCI, Subtarget);
24434 }
24435 case ISD::SUB: {
24436 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
24437 return V;
24438 return performSUBCombine(N, DAG, Subtarget);
24439 }
24440 case ISD::AND:
24441 return performANDCombine(N, DCI, Subtarget);
24442 case ISD::OR: {
24443 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
24444 return V;
24445 return performORCombine(N, DCI, Subtarget);
24446 }
24447 case ISD::XOR:
24448 return performXORCombine(N, DAG, Subtarget);
24449 case ISD::MUL:
24450 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
24451 return V;
24452 return performMULCombine(N, DAG, DCI, Subtarget);
24453 case ISD::SDIV:
24454 case ISD::UDIV:
24455 case ISD::SREM:
24456 case ISD::UREM:
24457 if (SDValue V = combineBinOpOfZExt(N, DAG))
24458 return V;
24459 break;
24460 case ISD::FMUL: {
24461 using namespace SDPatternMatch;
24462 SDLoc DL(N);
24463 EVT VT = N->getValueType(ResNo: 0);
24464 SDValue X, Y;
24465 // InstCombine canonicalizes fneg (fmul x, y) -> fmul x, (fneg y), see
24466 // hoistFNegAboveFMulFDiv.
24467 // Undo this and sink the fneg so we match more fmsub/fnmadd patterns.
24468 if (sd_match(N, P: m_FMul(L: m_Value(N&: X), R: m_OneUse(P: m_FNeg(Op: m_Value(N&: Y))))))
24469 return DAG.getNode(Opcode: ISD::FNEG, DL, VT,
24470 Operand: DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: Y, Flags: N->getFlags()),
24471 Flags: N->getFlags());
24472
24473 // fmul X, (copysign 1.0, Y) -> fsgnjx X, Y
24474 SDValue N0 = N->getOperand(Num: 0);
24475 SDValue N1 = N->getOperand(Num: 1);
24476 if (N0->getOpcode() != ISD::FCOPYSIGN)
24477 std::swap(a&: N0, b&: N1);
24478 if (N0->getOpcode() != ISD::FCOPYSIGN)
24479 return SDValue();
24480 ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: N0->getOperand(Num: 0));
24481 if (!C || !C->getValueAPF().isOne())
24482 return SDValue();
24483 if (VT.isVector() || !isOperationLegal(Op: ISD::FCOPYSIGN, VT))
24484 return SDValue();
24485 SDValue Sign = N0->getOperand(Num: 1);
24486 if (Sign.getValueType() != VT)
24487 return SDValue();
24488 return DAG.getNode(Opcode: RISCVISD::FSGNJX, DL, VT, N1, N2: N0->getOperand(Num: 1));
24489 }
24490 case ISD::UMAX:
24491 case ISD::UMIN:
24492 case ISD::SMAX:
24493 case ISD::SMIN:
24494 if (SDValue V = combineMinMaxToSat(N, DCI, Subtarget))
24495 return V;
24496 [[fallthrough]];
24497 case ISD::FADD:
24498 case ISD::FMAXNUM:
24499 case ISD::FMINNUM: {
24500 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
24501 return V;
24502 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
24503 return V;
24504 return SDValue();
24505 }
24506 case ISD::FMA: {
24507 SDValue N0 = N->getOperand(Num: 0);
24508 SDValue N1 = N->getOperand(Num: 1);
24509 if (N0.getOpcode() != ISD::SPLAT_VECTOR)
24510 std::swap(a&: N0, b&: N1);
24511 if (N0.getOpcode() != ISD::SPLAT_VECTOR)
24512 return SDValue();
24513 SDValue SplatN0 = N0.getOperand(i: 0);
24514 if (SplatN0.getOpcode() != ISD::FNEG || !SplatN0.hasOneUse())
24515 return SDValue();
24516 EVT VT = N->getValueType(ResNo: 0);
24517 SDValue Splat =
24518 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: SplatN0.getOperand(i: 0));
24519 SDValue Fneg = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Splat);
24520 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Fneg, N2: N1, N3: N->getOperand(Num: 2));
24521 }
24522 case ISD::VECTOR_MATCH:
24523 if (DCI.isBeforeLegalize())
24524 return expandVectorMatch(N, DAG);
24525 return SDValue();
24526 case ISD::SETCC:
24527 return performSETCCCombine(N, DCI, Subtarget);
24528 case ISD::SIGN_EXTEND_INREG:
24529 return performSIGN_EXTEND_INREGCombine(N, DCI, Subtarget);
24530 case ISD::ZERO_EXTEND:
24531 // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
24532 // type legalization. This is safe because fp_to_uint produces poison if
24533 // it overflows.
24534 if (N->getValueType(ResNo: 0) == MVT::i64 && Subtarget.is64Bit()) {
24535 SDValue Src = N->getOperand(Num: 0);
24536 if (Src.getOpcode() == ISD::FP_TO_UINT &&
24537 isTypeLegal(VT: Src.getOperand(i: 0).getValueType()))
24538 return DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SDLoc(N), VT: MVT::i64,
24539 Operand: Src.getOperand(i: 0));
24540 if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
24541 isTypeLegal(VT: Src.getOperand(i: 1).getValueType())) {
24542 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other);
24543 SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_TO_UINT, DL: SDLoc(N), VTList: VTs,
24544 N1: Src.getOperand(i: 0), N2: Src.getOperand(i: 1));
24545 DCI.CombineTo(N, Res);
24546 DAG.ReplaceAllUsesOfValueWith(From: Src.getValue(R: 1), To: Res.getValue(R: 1));
24547 DCI.recursivelyDeleteUnusedNodes(N: Src.getNode());
24548 return SDValue(N, 0); // Return N so it doesn't get rechecked.
24549 }
24550 }
24551 return SDValue();
24552 case RISCVISD::TRUNCATE_VECTOR_VL:
24553 return combineTruncOfSraSext(N, DAG);
24554 case ISD::TRUNCATE:
24555 return performTRUNCATECombine(N, DAG, Subtarget);
24556 case ISD::SELECT:
24557 return performSELECTCombine(N, DAG, Subtarget);
24558 case ISD::VSELECT:
24559 return performVSELECTCombine(N, DAG, Subtarget);
24560 case RISCVISD::CZERO_EQZ:
24561 case RISCVISD::CZERO_NEZ: {
24562 SDValue Val = N->getOperand(Num: 0);
24563 SDValue Cond = N->getOperand(Num: 1);
24564 MVT VT = N->getSimpleValueType(ResNo: 0);
24565
24566 unsigned Opc = N->getOpcode();
24567
24568 // czero_eqz x, x -> x
24569 if (Opc == RISCVISD::CZERO_EQZ && Val == Cond)
24570 return Val;
24571
24572 unsigned InvOpc =
24573 Opc == RISCVISD::CZERO_EQZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ;
24574
24575 // czero_eqz X, (xor Y, 1) -> czero_nez X, Y if Y is 0 or 1.
24576 // czero_nez X, (xor Y, 1) -> czero_eqz X, Y if Y is 0 or 1.
24577 if (Cond.getOpcode() == ISD::XOR && isOneConstant(V: Cond.getOperand(i: 1))) {
24578 SDValue NewCond = Cond.getOperand(i: 0);
24579 APInt Mask = APInt::getBitsSetFrom(numBits: NewCond.getValueSizeInBits(), loBit: 1);
24580 if (DAG.MaskedValueIsZero(Op: NewCond, Mask))
24581 return DAG.getNode(Opcode: InvOpc, DL: SDLoc(N), VT, N1: Val, N2: NewCond);
24582 }
24583 // czero_eqz x, (setcc y, 0, ne) -> czero_eqz x, y
24584 // czero_nez x, (setcc y, 0, ne) -> czero_nez x, y
24585 // czero_eqz x, (setcc y, 0, eq) -> czero_nez x, y
24586 // czero_nez x, (setcc y, 0, eq) -> czero_eqz x, y
24587 if (Cond.getOpcode() == ISD::SETCC && isNullConstant(V: Cond.getOperand(i: 1))) {
24588 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
24589 if (ISD::isIntEqualitySetCC(Code: CCVal))
24590 return DAG.getNode(Opcode: CCVal == ISD::SETNE ? Opc : InvOpc, DL: SDLoc(N), VT,
24591 N1: Val, N2: Cond.getOperand(i: 0));
24592 }
24593
24594 // Remove SRL from bittest patterns (srl (and X, (1 << C)), C) if the and
24595 // is an ANDI. Because only 1 bit can be set after the AND, it doesn't
24596 // matter if we shift it.
24597 if (Cond.getOpcode() == ISD::SRL &&
24598 isa<ConstantSDNode>(Val: Cond.getOperand(i: 1)) &&
24599 Cond.getOperand(i: 0).getOpcode() == ISD::AND) {
24600 const APInt &ShAmt = Cond.getConstantOperandAPInt(i: 1);
24601 unsigned BitWidth = VT.getSizeInBits();
24602 SDValue And = Cond.getOperand(i: 0);
24603 if (ShAmt.ult(RHS: BitWidth) && isa<ConstantSDNode>(Val: And.getOperand(i: 1))) {
24604 uint64_t AndConst = And.getConstantOperandVal(i: 1);
24605 if (AndConst == (1ULL << ShAmt.getZExtValue()) && isInt<12>(x: AndConst))
24606 return DAG.getNode(Opcode: Opc, DL, VT, N1: Val, N2: And);
24607 }
24608 }
24609
24610 // czero_nez (setcc X, Y, CC), (setcc X, Y, eq) -> (setcc X, Y, CC)
24611 // if CC is a strict inequality (lt, gt, ult, ugt), because when X == Y
24612 // the setcc result is already 0. The eq operands can be in either order.
24613 if (Opc == RISCVISD::CZERO_NEZ && Val.getOpcode() == ISD::SETCC &&
24614 Cond.getOpcode() == ISD::SETCC &&
24615 cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get() == ISD::SETEQ) {
24616 ISD::CondCode ValCC = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
24617 bool SameOperands = (Val.getOperand(i: 0) == Cond.getOperand(i: 0) &&
24618 Val.getOperand(i: 1) == Cond.getOperand(i: 1)) ||
24619 (Val.getOperand(i: 0) == Cond.getOperand(i: 1) &&
24620 Val.getOperand(i: 1) == Cond.getOperand(i: 0));
24621 if (SameOperands && (ValCC == ISD::SETLT || ValCC == ISD::SETGT ||
24622 ValCC == ISD::SETULT || ValCC == ISD::SETUGT))
24623 return Val;
24624 }
24625
24626 return SDValue();
24627 }
24628 case RISCVISD::SELECT_CC: {
24629 // Transform
24630 SDValue LHS = N->getOperand(Num: 0);
24631 SDValue RHS = N->getOperand(Num: 1);
24632 SDValue CC = N->getOperand(Num: 2);
24633 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
24634 SDValue TrueV = N->getOperand(Num: 3);
24635 SDValue FalseV = N->getOperand(Num: 4);
24636 SDLoc DL(N);
24637 EVT VT = N->getValueType(ResNo: 0);
24638
24639 // If the True and False values are the same, we don't need a select_cc.
24640 if (TrueV == FalseV)
24641 return TrueV;
24642
24643 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
24644 // (select (x >= 0), y, z) -> x >> (XLEN - 1) & (z - y) + y
24645 if (!Subtarget.hasShortForwardBranchIALU() && isa<ConstantSDNode>(Val: TrueV) &&
24646 isa<ConstantSDNode>(Val: FalseV) && isNullConstant(V: RHS) &&
24647 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
24648 if (CCVal == ISD::CondCode::SETGE)
24649 std::swap(a&: TrueV, b&: FalseV);
24650
24651 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
24652 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
24653 // Only handle simm12, if it is not in this range, it can be considered as
24654 // register.
24655 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
24656 isInt<12>(x: TrueSImm - FalseSImm)) {
24657 SDValue SRA =
24658 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
24659 N2: DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT));
24660 SDValue AND =
24661 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
24662 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
24663 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
24664 }
24665
24666 if (CCVal == ISD::CondCode::SETGE)
24667 std::swap(a&: TrueV, b&: FalseV);
24668 }
24669
24670 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
24671 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
24672 Ops: {LHS, RHS, CC, TrueV, FalseV});
24673
24674 if (!Subtarget.hasConditionalMoveFusion()) {
24675 // (select c, -1, y) -> -c | y
24676 if (isAllOnesConstant(V: TrueV)) {
24677 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: CCVal);
24678 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
24679 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: FalseV);
24680 }
24681 // (select c, y, -1) -> -!c | y
24682 if (isAllOnesConstant(V: FalseV)) {
24683 SDValue C =
24684 DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::getSetCCInverse(Operation: CCVal, Type: VT));
24685 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
24686 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: TrueV);
24687 }
24688
24689 // (select c, 0, y) -> -!c & y
24690 if (isNullConstant(V: TrueV)) {
24691 SDValue C =
24692 DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::getSetCCInverse(Operation: CCVal, Type: VT));
24693 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
24694 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: FalseV);
24695 }
24696 // (select c, y, 0) -> -c & y
24697 if (isNullConstant(V: FalseV)) {
24698 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: CCVal);
24699 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
24700 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: TrueV);
24701 }
24702 // (riscvisd::select_cc x, 0, ne, x, 1) -> (add x, (setcc x, 0, eq))
24703 // (riscvisd::select_cc x, 0, eq, 1, x) -> (add x, (setcc x, 0, eq))
24704 if (((isOneConstant(V: FalseV) && LHS == TrueV &&
24705 CCVal == ISD::CondCode::SETNE) ||
24706 (isOneConstant(V: TrueV) && LHS == FalseV &&
24707 CCVal == ISD::CondCode::SETEQ)) &&
24708 isNullConstant(V: RHS)) {
24709 // freeze it to be safe.
24710 LHS = DAG.getFreeze(V: LHS);
24711 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::CondCode::SETEQ);
24712 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: C);
24713 }
24714 }
24715
24716 // If both true/false are an xor with 1, pull through the select.
24717 // This can occur after op legalization if both operands are setccs that
24718 // require an xor to invert.
24719 // FIXME: Generalize to other binary ops with identical operand?
24720 if (TrueV.getOpcode() == ISD::XOR && FalseV.getOpcode() == ISD::XOR &&
24721 TrueV.getOperand(i: 1) == FalseV.getOperand(i: 1) &&
24722 isOneConstant(V: TrueV.getOperand(i: 1)) &&
24723 TrueV.hasOneUse() && FalseV.hasOneUse()) {
24724 SDValue NewSel = DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, N1: LHS, N2: RHS, N3: CC,
24725 N4: TrueV.getOperand(i: 0), N5: FalseV.getOperand(i: 0));
24726 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewSel, N2: TrueV.getOperand(i: 1));
24727 }
24728
24729 return SDValue();
24730 }
24731 case RISCVISD::BR_CC: {
24732 SDValue LHS = N->getOperand(Num: 1);
24733 SDValue RHS = N->getOperand(Num: 2);
24734 SDValue CC = N->getOperand(Num: 3);
24735 SDLoc DL(N);
24736
24737 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
24738 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
24739 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
24740
24741 return SDValue();
24742 }
24743 case ISD::BITREVERSE:
24744 return performBITREVERSECombine(N, DAG, Subtarget);
24745 case ISD::FP_TO_SINT:
24746 case ISD::FP_TO_UINT:
24747 return performFP_TO_INTCombine(N, DCI, Subtarget);
24748 case ISD::FP_TO_SINT_SAT:
24749 case ISD::FP_TO_UINT_SAT:
24750 return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
24751 case ISD::FCOPYSIGN: {
24752 EVT VT = N->getValueType(ResNo: 0);
24753 if (!VT.isVector())
24754 break;
24755 // There is a form of VFSGNJ which injects the negated sign of its second
24756 // operand. Try and bubble any FNEG up after the extend/round to produce
24757 // this optimized pattern. Avoid modifying cases where FP_ROUND and
24758 // TRUNC=1.
24759 SDValue In2 = N->getOperand(Num: 1);
24760 // Avoid cases where the extend/round has multiple uses, as duplicating
24761 // those is typically more expensive than removing a fneg.
24762 if (!In2.hasOneUse())
24763 break;
24764 if (In2.getOpcode() != ISD::FP_EXTEND &&
24765 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(i: 1) != 0))
24766 break;
24767 In2 = In2.getOperand(i: 0);
24768 if (In2.getOpcode() != ISD::FNEG)
24769 break;
24770 SDLoc DL(N);
24771 SDValue NewFPExtRound = DAG.getFPExtendOrRound(Op: In2.getOperand(i: 0), DL, VT);
24772 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT, N1: N->getOperand(Num: 0),
24773 N2: DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: NewFPExtRound));
24774 }
24775 case ISD::MGATHER: {
24776 const auto *MGN = cast<MaskedGatherSDNode>(Val: N);
24777 const EVT VT = N->getValueType(ResNo: 0);
24778 SDValue Index = MGN->getIndex();
24779 SDValue ScaleOp = MGN->getScale();
24780 ISD::MemIndexType IndexType = MGN->getIndexType();
24781 assert(!MGN->isIndexScaled() &&
24782 "Scaled gather/scatter should not be formed");
24783
24784 SDLoc DL(N);
24785 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
24786 return DAG.getMaskedGather(
24787 VTs: N->getVTList(), MemVT: MGN->getMemoryVT(), dl: DL,
24788 Ops: {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
24789 MGN->getBasePtr(), Index, ScaleOp},
24790 MMO: MGN->getMemOperand(), IndexType, ExtTy: MGN->getExtensionType());
24791
24792 if (narrowIndex(N&: Index, IndexType, DAG))
24793 return DAG.getMaskedGather(
24794 VTs: N->getVTList(), MemVT: MGN->getMemoryVT(), dl: DL,
24795 Ops: {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
24796 MGN->getBasePtr(), Index, ScaleOp},
24797 MMO: MGN->getMemOperand(), IndexType, ExtTy: MGN->getExtensionType());
24798
24799 if (Index.getOpcode() == ISD::BUILD_VECTOR &&
24800 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) {
24801 // The sequence will be XLenVT, not the type of Index. Tell
24802 // isSimpleVIDSequence this so we avoid overflow.
24803 if (std::optional<VIDSequence> SimpleVID =
24804 isSimpleVIDSequence(Op: Index, EltSizeInBits: Subtarget.getXLen());
24805 SimpleVID && SimpleVID->StepDenominator == 1) {
24806 const int64_t StepNumerator = SimpleVID->StepNumerator;
24807 const int64_t Addend = SimpleVID->Addend;
24808
24809 // Note: We don't need to check alignment here since (by assumption
24810 // from the existence of the gather), our offsets must be sufficiently
24811 // aligned.
24812
24813 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
24814 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
24815 assert(IndexType == ISD::UNSIGNED_SCALED);
24816 SDValue BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: MGN->getBasePtr(),
24817 N2: DAG.getSignedConstant(Val: Addend, DL, VT: PtrVT));
24818
24819 SDValue EVL = DAG.getElementCount(DL, VT: Subtarget.getXLenVT(),
24820 EC: VT.getVectorElementCount());
24821 SDValue StridedLoad = DAG.getStridedLoadVP(
24822 VT, DL, Chain: MGN->getChain(), Ptr: BasePtr,
24823 Stride: DAG.getSignedConstant(Val: StepNumerator, DL, VT: XLenVT), Mask: MGN->getMask(),
24824 EVL, MMO: MGN->getMemOperand());
24825 SDValue Select = DAG.getSelect(DL, VT, Cond: MGN->getMask(), LHS: StridedLoad,
24826 RHS: MGN->getPassThru());
24827 return DAG.getMergeValues(Ops: {Select, SDValue(StridedLoad.getNode(), 1)},
24828 dl: DL);
24829 }
24830 }
24831
24832 SmallVector<int> ShuffleMask;
24833 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
24834 matchIndexAsShuffle(VT, Index, Mask: MGN->getMask(), ShuffleMask)) {
24835 SDValue Load = DAG.getMaskedLoad(
24836 VT, dl: DL, Chain: MGN->getChain(), Base: MGN->getBasePtr(), Offset: DAG.getPOISON(VT: XLenVT),
24837 Mask: MGN->getMask(), Src0: DAG.getPOISON(VT), MemVT: MGN->getMemoryVT(),
24838 MMO: MGN->getMemOperand(), AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
24839 SDValue Shuffle =
24840 DAG.getVectorShuffle(VT, dl: DL, N1: Load, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
24841 return DAG.getMergeValues(Ops: {Shuffle, Load.getValue(R: 1)}, dl: DL);
24842 }
24843
24844 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
24845 matchIndexAsWiderOp(VT, Index, Mask: MGN->getMask(),
24846 BaseAlign: MGN->getMemOperand()->getBaseAlign(), ST: Subtarget)) {
24847 SmallVector<SDValue> NewIndices;
24848 for (unsigned i = 0; i < Index->getNumOperands(); i += 2)
24849 NewIndices.push_back(Elt: Index.getOperand(i));
24850 EVT IndexVT = Index.getValueType()
24851 .getHalfNumVectorElementsVT(Context&: *DAG.getContext());
24852 Index = DAG.getBuildVector(VT: IndexVT, DL, Ops: NewIndices);
24853
24854 unsigned ElementSize = VT.getScalarStoreSize();
24855 EVT WideScalarVT = MVT::getIntegerVT(BitWidth: ElementSize * 8 * 2);
24856 auto EltCnt = VT.getVectorElementCount();
24857 assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!");
24858 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideScalarVT,
24859 EC: EltCnt.divideCoefficientBy(RHS: 2));
24860 SDValue Passthru = DAG.getBitcast(VT: WideVT, V: MGN->getPassThru());
24861 EVT MaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
24862 EC: EltCnt.divideCoefficientBy(RHS: 2));
24863 SDValue Mask = DAG.getSplat(VT: MaskVT, DL, Op: DAG.getConstant(Val: 1, DL, VT: MVT::i1));
24864
24865 SDValue Gather =
24866 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), MemVT: WideVT, dl: DL,
24867 Ops: {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
24868 Index, ScaleOp},
24869 MMO: MGN->getMemOperand(), IndexType, ExtTy: ISD::NON_EXTLOAD);
24870 SDValue Result = DAG.getBitcast(VT, V: Gather.getValue(R: 0));
24871 return DAG.getMergeValues(Ops: {Result, Gather.getValue(R: 1)}, dl: DL);
24872 }
24873 break;
24874 }
24875 case ISD::MSCATTER:{
24876 const auto *MSN = cast<MaskedScatterSDNode>(Val: N);
24877 SDValue Index = MSN->getIndex();
24878 SDValue ScaleOp = MSN->getScale();
24879 ISD::MemIndexType IndexType = MSN->getIndexType();
24880 assert(!MSN->isIndexScaled() &&
24881 "Scaled gather/scatter should not be formed");
24882
24883 SDLoc DL(N);
24884 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
24885 return DAG.getMaskedScatter(
24886 VTs: N->getVTList(), MemVT: MSN->getMemoryVT(), dl: DL,
24887 Ops: {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
24888 Index, ScaleOp},
24889 MMO: MSN->getMemOperand(), IndexType, IsTruncating: MSN->isTruncatingStore());
24890
24891 if (narrowIndex(N&: Index, IndexType, DAG))
24892 return DAG.getMaskedScatter(
24893 VTs: N->getVTList(), MemVT: MSN->getMemoryVT(), dl: DL,
24894 Ops: {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
24895 Index, ScaleOp},
24896 MMO: MSN->getMemOperand(), IndexType, IsTruncating: MSN->isTruncatingStore());
24897
24898 EVT VT = MSN->getValue()->getValueType(ResNo: 0);
24899 SmallVector<int> ShuffleMask;
24900 if (!MSN->isTruncatingStore() &&
24901 matchIndexAsShuffle(VT, Index, Mask: MSN->getMask(), ShuffleMask)) {
24902 SDValue Shuffle = DAG.getVectorShuffle(VT, dl: DL, N1: MSN->getValue(),
24903 N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
24904 return DAG.getMaskedStore(Chain: MSN->getChain(), dl: DL, Val: Shuffle, Base: MSN->getBasePtr(),
24905 Offset: DAG.getPOISON(VT: XLenVT), Mask: MSN->getMask(),
24906 MemVT: MSN->getMemoryVT(), MMO: MSN->getMemOperand(),
24907 AM: ISD::UNINDEXED, IsTruncating: false);
24908 }
24909 break;
24910 }
24911 case ISD::VP_GATHER: {
24912 const auto *VPGN = cast<VPGatherSDNode>(Val: N);
24913 SDValue Index = VPGN->getIndex();
24914 SDValue ScaleOp = VPGN->getScale();
24915 ISD::MemIndexType IndexType = VPGN->getIndexType();
24916 assert(!VPGN->isIndexScaled() &&
24917 "Scaled gather/scatter should not be formed");
24918
24919 SDLoc DL(N);
24920 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
24921 return DAG.getGatherVP(VTs: N->getVTList(), VT: VPGN->getMemoryVT(), dl: DL,
24922 Ops: {VPGN->getChain(), VPGN->getBasePtr(), Index,
24923 ScaleOp, VPGN->getMask(),
24924 VPGN->getVectorLength()},
24925 MMO: VPGN->getMemOperand(), IndexType);
24926
24927 if (narrowIndex(N&: Index, IndexType, DAG))
24928 return DAG.getGatherVP(VTs: N->getVTList(), VT: VPGN->getMemoryVT(), dl: DL,
24929 Ops: {VPGN->getChain(), VPGN->getBasePtr(), Index,
24930 ScaleOp, VPGN->getMask(),
24931 VPGN->getVectorLength()},
24932 MMO: VPGN->getMemOperand(), IndexType);
24933
24934 break;
24935 }
24936 case ISD::VP_SCATTER: {
24937 const auto *VPSN = cast<VPScatterSDNode>(Val: N);
24938 SDValue Index = VPSN->getIndex();
24939 SDValue ScaleOp = VPSN->getScale();
24940 ISD::MemIndexType IndexType = VPSN->getIndexType();
24941 assert(!VPSN->isIndexScaled() &&
24942 "Scaled gather/scatter should not be formed");
24943
24944 SDLoc DL(N);
24945 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
24946 return DAG.getScatterVP(VTs: N->getVTList(), VT: VPSN->getMemoryVT(), dl: DL,
24947 Ops: {VPSN->getChain(), VPSN->getValue(),
24948 VPSN->getBasePtr(), Index, ScaleOp,
24949 VPSN->getMask(), VPSN->getVectorLength()},
24950 MMO: VPSN->getMemOperand(), IndexType);
24951
24952 if (narrowIndex(N&: Index, IndexType, DAG))
24953 return DAG.getScatterVP(VTs: N->getVTList(), VT: VPSN->getMemoryVT(), dl: DL,
24954 Ops: {VPSN->getChain(), VPSN->getValue(),
24955 VPSN->getBasePtr(), Index, ScaleOp,
24956 VPSN->getMask(), VPSN->getVectorLength()},
24957 MMO: VPSN->getMemOperand(), IndexType);
24958 break;
24959 }
24960 case RISCVISD::SHL_VL:
24961 if (SDValue V = performSHLCombine(N, DCI, Subtarget))
24962 return V;
24963 [[fallthrough]];
24964 case RISCVISD::SRA_VL:
24965 case RISCVISD::SRL_VL: {
24966 SDValue ShAmt = N->getOperand(Num: 1);
24967 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
24968 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
24969 SDLoc DL(N);
24970 SDValue VL = N->getOperand(Num: 4);
24971 EVT VT = N->getValueType(ResNo: 0);
24972 ShAmt = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
24973 N2: ShAmt.getOperand(i: 1), N3: VL);
24974 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N->getOperand(Num: 0), N2: ShAmt,
24975 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
24976 }
24977 break;
24978 }
24979 case ISD::SRA:
24980 if (SDValue V = performSRACombine(N, DAG, Subtarget))
24981 return V;
24982 [[fallthrough]];
24983 case ISD::SRL:
24984 case ISD::SHL: {
24985 if (N->getOpcode() == ISD::SHL) {
24986 if (SDValue V = performSHLCombine(N, DCI, Subtarget))
24987 return V;
24988 }
24989 SDValue ShAmt = N->getOperand(Num: 1);
24990 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
24991 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
24992 SDLoc DL(N);
24993 EVT VT = N->getValueType(ResNo: 0);
24994 ShAmt = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
24995 N2: ShAmt.getOperand(i: 1),
24996 N3: DAG.getRegister(Reg: RISCV::X0, VT: Subtarget.getXLenVT()));
24997 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N->getOperand(Num: 0), N2: ShAmt);
24998 }
24999 break;
25000 }
25001 case RISCVISD::ADD_VL:
25002 if (SDValue V = simplifyOp_VL(N))
25003 return V;
25004 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
25005 return V;
25006 if (SDValue V = combineVdot4aAccum(N, DAG, Subtarget))
25007 return V;
25008 return combineToVWMACC(N, DAG, Subtarget);
25009 case RISCVISD::VWADDU_VL:
25010 return performVWABDACombine(N, DAG, Subtarget);
25011 case RISCVISD::VWADDU_W_VL:
25012 if (SDValue V = performVWABDACombineWV(N, DAG, Subtarget))
25013 return V;
25014 [[fallthrough]];
25015 case RISCVISD::VWADD_W_VL:
25016 case RISCVISD::VWSUB_W_VL:
25017 case RISCVISD::VWSUBU_W_VL:
25018 return performVWADDSUBW_VLCombine(N, DCI, Subtarget);
25019 case RISCVISD::OR_VL:
25020 case RISCVISD::SUB_VL:
25021 case RISCVISD::MUL_VL:
25022 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
25023 case RISCVISD::VFMADD_VL:
25024 case RISCVISD::VFNMADD_VL:
25025 case RISCVISD::VFMSUB_VL:
25026 case RISCVISD::VFNMSUB_VL:
25027 case RISCVISD::STRICT_VFMADD_VL:
25028 case RISCVISD::STRICT_VFNMADD_VL:
25029 case RISCVISD::STRICT_VFMSUB_VL:
25030 case RISCVISD::STRICT_VFNMSUB_VL:
25031 return performVFMADD_VLCombine(N, DCI, Subtarget);
25032 case RISCVISD::FADD_VL:
25033 case RISCVISD::FSUB_VL:
25034 case RISCVISD::FMUL_VL:
25035 case RISCVISD::VFWADD_W_VL:
25036 case RISCVISD::VFWSUB_W_VL:
25037 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
25038 case RISCVISD::VSEXT_VL:
25039 case RISCVISD::VZEXT_VL:
25040 return performVEXT_VLCombine(N, DCI, Subtarget);
25041 case ISD::LOAD:
25042 case ISD::STORE: {
25043 if (DCI.isAfterLegalizeDAG())
25044 if (SDValue V = performMemPairCombine(N, DCI))
25045 return V;
25046
25047 if (N->getOpcode() != ISD::STORE)
25048 break;
25049
25050 auto *Store = cast<StoreSDNode>(Val: N);
25051 SDValue Chain = Store->getChain();
25052 EVT MemVT = Store->getMemoryVT();
25053 SDValue Val = Store->getValue();
25054 SDLoc DL(N);
25055
25056 bool IsScalarizable =
25057 MemVT.isFixedLengthVector() && ISD::isNormalStore(N: Store) &&
25058 Store->isSimple() &&
25059 MemVT.getVectorElementType().bitsLE(VT: Subtarget.getXLenVT()) &&
25060 isPowerOf2_64(Value: MemVT.getSizeInBits()) &&
25061 MemVT.getSizeInBits() <= Subtarget.getXLen();
25062
25063 // If sufficiently aligned we can scalarize stores of constant vectors of
25064 // any power-of-two size up to XLen bits, provided that they aren't too
25065 // expensive to materialize.
25066 // vsetivli zero, 2, e8, m1, ta, ma
25067 // vmv.v.i v8, 4
25068 // vse64.v v8, (a0)
25069 // ->
25070 // li a1, 1028
25071 // sh a1, 0(a0)
25072 if (DCI.isBeforeLegalize() && IsScalarizable &&
25073 ISD::isBuildVectorOfConstantSDNodes(N: Val.getNode())) {
25074 // Get the constant vector bits
25075 APInt NewC(Val.getValueSizeInBits(), 0);
25076 uint64_t EltSize = Val.getScalarValueSizeInBits();
25077 for (unsigned i = 0; i < Val.getNumOperands(); i++) {
25078 if (Val.getOperand(i).isUndef())
25079 continue;
25080 NewC.insertBits(SubBits: Val.getConstantOperandAPInt(i).trunc(width: EltSize),
25081 bitPosition: i * EltSize);
25082 }
25083 MVT NewVT = MVT::getIntegerVT(BitWidth: MemVT.getSizeInBits());
25084
25085 if (RISCVMatInt::getIntMatCost(Val: NewC, Size: Subtarget.getXLen(), STI: Subtarget,
25086 CompressionCost: true) <= 2 &&
25087 allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
25088 VT: NewVT, MMO: *Store->getMemOperand())) {
25089 SDValue NewV = DAG.getConstant(Val: NewC, DL, VT: NewVT);
25090 return DAG.getStore(Chain, dl: DL, Val: NewV, Ptr: Store->getBasePtr(),
25091 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
25092 MMOFlags: Store->getMemOperand()->getFlags());
25093 }
25094 }
25095
25096 // Similarly, if sufficiently aligned we can scalarize vector copies, e.g.
25097 // vsetivli zero, 2, e16, m1, ta, ma
25098 // vle16.v v8, (a0)
25099 // vse16.v v8, (a1)
25100 if (auto *L = dyn_cast<LoadSDNode>(Val);
25101 L && DCI.isBeforeLegalize() && IsScalarizable && L->isSimple() &&
25102 L->hasNUsesOfValue(NUses: 1, Value: 0) && L->hasNUsesOfValue(NUses: 1, Value: 1) &&
25103 Store->getChain() == SDValue(L, 1) && ISD::isNormalLoad(N: L) &&
25104 L->getMemoryVT() == MemVT) {
25105 MVT NewVT = MVT::getIntegerVT(BitWidth: MemVT.getSizeInBits());
25106 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
25107 VT: NewVT, MMO: *Store->getMemOperand()) &&
25108 allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
25109 VT: NewVT, MMO: *L->getMemOperand())) {
25110 SDValue NewL = DAG.getLoad(VT: NewVT, dl: DL, Chain: L->getChain(), Ptr: L->getBasePtr(),
25111 PtrInfo: L->getPointerInfo(), Alignment: L->getBaseAlign(),
25112 MMOFlags: L->getMemOperand()->getFlags());
25113 return DAG.getStore(Chain, dl: DL, Val: NewL, Ptr: Store->getBasePtr(),
25114 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
25115 MMOFlags: Store->getMemOperand()->getFlags());
25116 }
25117 }
25118
25119 // Combine store of vmv.x.s/vfmv.f.s to vse with VL of 1.
25120 // vfmv.f.s is represented as extract element from 0. Match it late to avoid
25121 // any illegal types.
25122 if ((Val.getOpcode() == RISCVISD::VMV_X_S ||
25123 (DCI.isAfterLegalizeDAG() &&
25124 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
25125 isNullConstant(V: Val.getOperand(i: 1)))) &&
25126 Val.hasOneUse()) {
25127 SDValue Src = Val.getOperand(i: 0);
25128 EVT VecVT = Src.getValueType();
25129 // VecVT should be scalable and memory VT should match the element type.
25130 if (!Store->isIndexed() && Store->isSimple() &&
25131 VecVT.isScalableVectorOf(EltVT: MemVT)) {
25132 SDLoc DL(N);
25133 MVT MaskVT = getMaskTypeFor(VecVT: VecVT.getSimpleVT());
25134 // Create a vector memory VT so allowsMisalignedMemoryAccesses will
25135 // work correctly.
25136 MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT, NumElements: 1);
25137 return DAG.getStoreVP(
25138 Chain: Store->getChain(), dl: DL, Val: Src, Ptr: Store->getBasePtr(), Offset: Store->getOffset(),
25139 Mask: DAG.getConstant(Val: 1, DL, VT: MaskVT),
25140 EVL: DAG.getConstant(Val: 1, DL, VT: Subtarget.getXLenVT()), MemVT,
25141 MMO: Store->getMemOperand(), AM: Store->getAddressingMode());
25142 }
25143 }
25144
25145 break;
25146 }
25147 case ISD::SPLAT_VECTOR: {
25148 EVT VT = N->getValueType(ResNo: 0);
25149 // Only perform this combine on legal MVT types.
25150 if (!isTypeLegal(VT))
25151 break;
25152 if (auto Gather = matchSplatAsGather(SplatVal: N->getOperand(Num: 0), VT: VT.getSimpleVT(), DL: N,
25153 DAG, Subtarget))
25154 return Gather;
25155 break;
25156 }
25157 case ISD::BUILD_VECTOR:
25158 if (SDValue V = performBUILD_VECTORCombine(N, DAG, Subtarget, TLI: *this))
25159 return V;
25160 break;
25161 case ISD::CONCAT_VECTORS:
25162 if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, TLI: *this))
25163 return V;
25164 break;
25165 case ISD::VECTOR_SHUFFLE:
25166 if (SDValue V = performVECTOR_SHUFFLECombine(N, DAG, Subtarget, TLI: *this))
25167 return V;
25168 break;
25169 case ISD::INSERT_VECTOR_ELT:
25170 if (SDValue V = performINSERT_VECTOR_ELTCombine(N, DAG, Subtarget, TLI: *this))
25171 return V;
25172 break;
25173 case RISCVISD::VFMV_V_F_VL: {
25174 const MVT VT = N->getSimpleValueType(ResNo: 0);
25175 SDValue Passthru = N->getOperand(Num: 0);
25176 SDValue Scalar = N->getOperand(Num: 1);
25177 SDValue VL = N->getOperand(Num: 2);
25178
25179 // If VL is 1, we can use vfmv.s.f.
25180 if (isOneConstant(V: VL))
25181 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
25182 break;
25183 }
25184 case RISCVISD::VMV_V_X_VL: {
25185 const MVT VT = N->getSimpleValueType(ResNo: 0);
25186 SDValue Passthru = N->getOperand(Num: 0);
25187 SDValue Scalar = N->getOperand(Num: 1);
25188 SDValue VL = N->getOperand(Num: 2);
25189
25190 // Tail agnostic VMV.V.X only demands the vector element bitwidth from the
25191 // scalar input.
25192 unsigned ScalarSize = Scalar.getValueSizeInBits();
25193 unsigned EltWidth = VT.getScalarSizeInBits();
25194 if (ScalarSize > EltWidth && Passthru.isUndef())
25195 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
25196 return SDValue(N, 0);
25197
25198 // If VL is 1 and the scalar value won't benefit from immediate, we can
25199 // use vmv.s.x.
25200 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: Scalar);
25201 if (isOneConstant(V: VL) &&
25202 (!Const || Const->isZero() ||
25203 !Const->getAPIntValue().sextOrTrunc(width: EltWidth).isSignedIntN(N: 5)))
25204 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
25205
25206 break;
25207 }
25208 case RISCVISD::VFMV_S_F_VL: {
25209 SDValue Src = N->getOperand(Num: 1);
25210 // Try to remove vector->scalar->vector if the scalar->vector is inserting
25211 // into an undef vector.
25212 // TODO: Could use a vslide or vmv.v.v for non-undef.
25213 if (N->getOperand(Num: 0).isUndef() &&
25214 Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
25215 isNullConstant(V: Src.getOperand(i: 1)) &&
25216 Src.getOperand(i: 0).getValueType().isScalableVector()) {
25217 EVT VT = N->getValueType(ResNo: 0);
25218 SDValue EVSrc = Src.getOperand(i: 0);
25219 EVT EVSrcVT = EVSrc.getValueType();
25220 assert(EVSrcVT.getVectorElementType() == VT.getVectorElementType());
25221 // Widths match, just return the original vector.
25222 if (EVSrcVT == VT)
25223 return EVSrc;
25224 SDLoc DL(N);
25225 // Width is narrower, using insert_subvector.
25226 if (EVSrcVT.getVectorMinNumElements() < VT.getVectorMinNumElements()) {
25227 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT),
25228 N2: EVSrc,
25229 N3: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
25230 }
25231 // Width is wider, using extract_subvector.
25232 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: EVSrc,
25233 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
25234 }
25235 [[fallthrough]];
25236 }
25237 case RISCVISD::VMV_S_X_VL: {
25238 const MVT VT = N->getSimpleValueType(ResNo: 0);
25239 SDValue Passthru = N->getOperand(Num: 0);
25240 SDValue Scalar = N->getOperand(Num: 1);
25241 SDValue VL = N->getOperand(Num: 2);
25242
25243 // The vmv.s.x instruction copies the scalar integer register to element 0
25244 // of the destination vector register. If SEW < XLEN, the least-significant
25245 // bits are copied and the upper XLEN-SEW bits are ignored.
25246 unsigned ScalarSize = Scalar.getValueSizeInBits();
25247 unsigned EltWidth = VT.getScalarSizeInBits();
25248 if (ScalarSize > EltWidth && SimplifyDemandedLowBitsHelper(1, EltWidth))
25249 return SDValue(N, 0);
25250
25251 if (Scalar.getOpcode() == RISCVISD::VMV_X_S && Passthru.isUndef() &&
25252 Scalar.getOperand(i: 0).getValueType() == N->getValueType(ResNo: 0))
25253 return Scalar.getOperand(i: 0);
25254
25255 // Use M1 or smaller to avoid over constraining register allocation
25256 const MVT M1VT = RISCVTargetLowering::getM1VT(VT);
25257 if (M1VT.bitsLT(VT)) {
25258 SDValue M1Passthru = DAG.getExtractSubvector(DL, VT: M1VT, Vec: Passthru, Idx: 0);
25259 SDValue Result =
25260 DAG.getNode(Opcode: N->getOpcode(), DL, VT: M1VT, N1: M1Passthru, N2: Scalar, N3: VL);
25261 Result = DAG.getInsertSubvector(DL, Vec: Passthru, SubVec: Result, Idx: 0);
25262 return Result;
25263 }
25264
25265 // We use a vmv.v.i if possible. We limit this to LMUL1. LMUL2 or
25266 // higher would involve overly constraining the register allocator for
25267 // no purpose.
25268 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: Scalar);
25269 Const && !Const->isZero() && isInt<5>(x: Const->getSExtValue()) &&
25270 VT.bitsLE(VT: RISCVTargetLowering::getM1VT(VT)) && Passthru.isUndef())
25271 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
25272
25273 break;
25274 }
25275 case RISCVISD::VMV_X_S: {
25276 SDValue Vec = N->getOperand(Num: 0);
25277 MVT VecVT = N->getOperand(Num: 0).getSimpleValueType();
25278 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: VecVT);
25279 if (M1VT.bitsLT(VT: VecVT)) {
25280 Vec = DAG.getExtractSubvector(DL, VT: M1VT, Vec, Idx: 0);
25281 return DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: N->getValueType(ResNo: 0), Operand: Vec);
25282 }
25283 break;
25284 }
25285 case ISD::INTRINSIC_VOID:
25286 case ISD::INTRINSIC_W_CHAIN:
25287 case ISD::INTRINSIC_WO_CHAIN: {
25288 unsigned IntOpNo = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
25289 unsigned IntNo = N->getConstantOperandVal(Num: IntOpNo);
25290 switch (IntNo) {
25291 // By default we do not combine any intrinsic.
25292 default:
25293 return SDValue();
25294 case Intrinsic::riscv_vcpop:
25295 case Intrinsic::riscv_vcpop_mask:
25296 case Intrinsic::riscv_vfirst:
25297 case Intrinsic::riscv_vfirst_mask: {
25298 SDValue VL = N->getOperand(Num: 2);
25299 if (IntNo == Intrinsic::riscv_vcpop_mask ||
25300 IntNo == Intrinsic::riscv_vfirst_mask)
25301 VL = N->getOperand(Num: 3);
25302 if (!isNullConstant(V: VL))
25303 return SDValue();
25304 // If VL is 0, vcpop -> li 0, vfirst -> li -1.
25305 SDLoc DL(N);
25306 EVT VT = N->getValueType(ResNo: 0);
25307 if (IntNo == Intrinsic::riscv_vfirst ||
25308 IntNo == Intrinsic::riscv_vfirst_mask)
25309 return DAG.getAllOnesConstant(DL, VT);
25310 return DAG.getConstant(Val: 0, DL, VT);
25311 }
25312 case Intrinsic::riscv_vsseg2_mask:
25313 case Intrinsic::riscv_vsseg3_mask:
25314 case Intrinsic::riscv_vsseg4_mask:
25315 case Intrinsic::riscv_vsseg5_mask:
25316 case Intrinsic::riscv_vsseg6_mask:
25317 case Intrinsic::riscv_vsseg7_mask:
25318 case Intrinsic::riscv_vsseg8_mask: {
25319 SDValue Tuple = N->getOperand(Num: 2);
25320 unsigned NF = Tuple.getValueType().getRISCVVectorTupleNumFields();
25321
25322 if (Subtarget.hasOptimizedSegmentLoadStore(NF) || !Tuple.hasOneUse() ||
25323 Tuple.getOpcode() != RISCVISD::TUPLE_INSERT ||
25324 !Tuple.getOperand(i: 0).isUndef())
25325 return SDValue();
25326
25327 SDValue Val = Tuple.getOperand(i: 1);
25328 unsigned Idx = Tuple.getConstantOperandVal(i: 2);
25329
25330 unsigned SEW = Val.getValueType().getScalarSizeInBits();
25331 assert(Log2_64(SEW) == N->getConstantOperandVal(6) &&
25332 "Type mismatch without bitcast?");
25333 unsigned Stride = SEW / 8 * NF;
25334 unsigned Offset = SEW / 8 * Idx;
25335
25336 SDValue Ops[] = {
25337 /*Chain=*/N->getOperand(Num: 0),
25338 /*IntID=*/
25339 DAG.getTargetConstant(Val: Intrinsic::riscv_vsse_mask, DL, VT: XLenVT),
25340 /*StoredVal=*/Val,
25341 /*Ptr=*/
25342 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: N->getOperand(Num: 3),
25343 N2: DAG.getConstant(Val: Offset, DL, VT: XLenVT)),
25344 /*Stride=*/DAG.getConstant(Val: Stride, DL, VT: XLenVT),
25345 /*Mask=*/N->getOperand(Num: 4),
25346 /*VL=*/N->getOperand(Num: 5)};
25347
25348 auto *OldMemSD = cast<MemIntrinsicSDNode>(Val: N);
25349 // Match getTgtMemIntrinsic for non-unit stride case
25350 EVT MemVT = OldMemSD->getMemoryVT().getScalarType();
25351 MachineFunction &MF = DAG.getMachineFunction();
25352 MachineMemOperand *MMO = MF.getMachineMemOperand(
25353 MMO: OldMemSD->getMemOperand(), Offset, Size: MemoryLocation::UnknownSize);
25354
25355 SDVTList VTs = DAG.getVTList(VT: MVT::Other);
25356 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: VTs, Ops, MemVT,
25357 MMO);
25358 }
25359 }
25360 }
25361 case ISD::VECTOR_SPLICE_RIGHT:
25362 case ISD::EXPERIMENTAL_VP_REVERSE:
25363 return performReverseEVLCombine(N, DCI, Subtarget);
25364 case ISD::VP_STORE:
25365 return performVP_STORECombine(N, DAG, Subtarget);
25366 case ISD::BITCAST: {
25367 if (Subtarget.hasStdExtP())
25368 if (SDValue V = performP_BITCASTCombine(N, DAG, Subtarget))
25369 return V;
25370 if (!Subtarget.useRVVForFixedLengthVectors())
25371 return SDValue();
25372 SDValue N0 = N->getOperand(Num: 0);
25373 EVT VT = N->getValueType(ResNo: 0);
25374 EVT SrcVT = N0.getValueType();
25375 if (VT.isRISCVVectorTuple() && N0->getOpcode() == ISD::SPLAT_VECTOR) {
25376 unsigned NF = VT.getRISCVVectorTupleNumFields();
25377 unsigned NumScalElts = VT.getSizeInBits().getKnownMinValue() / (NF * 8);
25378 SDValue EltVal = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
25379 MVT ScalTy = MVT::getScalableVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: NumScalElts);
25380
25381 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: ScalTy, Operand: EltVal);
25382
25383 SDValue Result = DAG.getUNDEF(VT);
25384 for (unsigned i = 0; i < NF; ++i)
25385 Result = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT, N1: Result, N2: Splat,
25386 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
25387 return Result;
25388 }
25389 // If this is a bitcast between a MVT::v4i1/v2i1/v1i1 and an illegal integer
25390 // type, widen both sides to avoid a trip through memory.
25391 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
25392 VT.isScalarInteger()) {
25393 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
25394 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT: SrcVT));
25395 Ops[0] = N0;
25396 SDLoc DL(N);
25397 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i1, Ops);
25398 N0 = DAG.getBitcast(VT: MVT::i8, V: N0);
25399 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0);
25400 }
25401
25402 return SDValue();
25403 }
25404 case ISD::VECREDUCE_ADD:
25405 if (SDValue V = performVECREDUCECombine(N, DAG, Subtarget, TLI: *this))
25406 return V;
25407 [[fallthrough]];
25408 case ISD::CTPOP:
25409 if (SDValue V = combineToVCPOP(N, DAG, Subtarget))
25410 return V;
25411 break;
25412 case RISCVISD::VRGATHER_VX_VL: {
25413 // Note this assumes that out of bounds indices produce poison
25414 // and can thus be replaced without having to prove them inbounds..
25415 EVT VT = N->getValueType(ResNo: 0);
25416 SDValue Src = N->getOperand(Num: 0);
25417 SDValue Idx = N->getOperand(Num: 1);
25418 SDValue Passthru = N->getOperand(Num: 2);
25419 SDValue VL = N->getOperand(Num: 4);
25420
25421 // Warning: Unlike most cases we strip an insert_subvector, this one
25422 // does not require the first operand to be undef.
25423 if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
25424 isNullConstant(V: Src.getOperand(i: 2)))
25425 Src = Src.getOperand(i: 1);
25426
25427 switch (Src.getOpcode()) {
25428 default:
25429 break;
25430 case RISCVISD::VMV_V_X_VL:
25431 case RISCVISD::VFMV_V_F_VL:
25432 // Drop a redundant vrgather_vx.
25433 // TODO: Remove the type restriction if we find a motivating
25434 // test case?
25435 if (Passthru.isUndef() && VL == Src.getOperand(i: 2) &&
25436 Src.getValueType() == VT)
25437 return Src;
25438 break;
25439 case RISCVISD::VMV_S_X_VL:
25440 case RISCVISD::VFMV_S_F_VL:
25441 // If this use only demands lane zero from the source vmv.s.x, and
25442 // doesn't have a passthru, then this vrgather.vi/vx is equivalent to
25443 // a vmv.v.x. Note that there can be other uses of the original
25444 // vmv.s.x and thus we can't eliminate it. (vfmv.s.f is analogous)
25445 if (isNullConstant(V: Idx) && Passthru.isUndef() &&
25446 VL == Src.getOperand(i: 2)) {
25447 unsigned Opc =
25448 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
25449 return DAG.getNode(Opcode: Opc, DL, VT, N1: DAG.getUNDEF(VT), N2: Src.getOperand(i: 1),
25450 N3: VL);
25451 }
25452 break;
25453 }
25454 break;
25455 }
25456 case RISCVISD::TUPLE_EXTRACT: {
25457 EVT VT = N->getValueType(ResNo: 0);
25458 SDValue Tuple = N->getOperand(Num: 0);
25459 unsigned Idx = N->getConstantOperandVal(Num: 1);
25460 if (!Tuple.hasOneUse() || Tuple.getOpcode() != ISD::INTRINSIC_W_CHAIN)
25461 break;
25462
25463 unsigned NF = 0;
25464 switch (Tuple.getConstantOperandVal(i: 1)) {
25465 default:
25466 break;
25467 case Intrinsic::riscv_vlseg2_mask:
25468 case Intrinsic::riscv_vlseg3_mask:
25469 case Intrinsic::riscv_vlseg4_mask:
25470 case Intrinsic::riscv_vlseg5_mask:
25471 case Intrinsic::riscv_vlseg6_mask:
25472 case Intrinsic::riscv_vlseg7_mask:
25473 case Intrinsic::riscv_vlseg8_mask:
25474 NF = Tuple.getValueType().getRISCVVectorTupleNumFields();
25475 break;
25476 }
25477
25478 if (!NF || Subtarget.hasOptimizedSegmentLoadStore(NF))
25479 break;
25480
25481 unsigned SEW = VT.getScalarSizeInBits();
25482 assert(Log2_64(SEW) == Tuple.getConstantOperandVal(7) &&
25483 "Type mismatch without bitcast?");
25484 unsigned Stride = SEW / 8 * NF;
25485 unsigned Offset = SEW / 8 * Idx;
25486
25487 SDValue Passthru = Tuple.getOperand(i: 2);
25488 if (Passthru.isUndef())
25489 Passthru = DAG.getUNDEF(VT);
25490 else
25491 Passthru = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT, N1: Passthru,
25492 N2: N->getOperand(Num: 1));
25493
25494 SDValue Ops[] = {
25495 /*Chain=*/Tuple.getOperand(i: 0),
25496 /*IntID=*/DAG.getTargetConstant(Val: Intrinsic::riscv_vlse_mask, DL, VT: XLenVT),
25497 /*Passthru=*/Passthru,
25498 /*Ptr=*/
25499 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: Tuple.getOperand(i: 3),
25500 N2: DAG.getConstant(Val: Offset, DL, VT: XLenVT)),
25501 /*Stride=*/DAG.getConstant(Val: Stride, DL, VT: XLenVT),
25502 /*Mask=*/Tuple.getOperand(i: 4),
25503 /*VL=*/Tuple.getOperand(i: 5),
25504 /*Policy=*/Tuple.getOperand(i: 6)};
25505
25506 auto *TupleMemSD = cast<MemIntrinsicSDNode>(Val&: Tuple);
25507 // Match getTgtMemIntrinsic for non-unit stride case
25508 EVT MemVT = TupleMemSD->getMemoryVT().getScalarType();
25509 MachineFunction &MF = DAG.getMachineFunction();
25510 MachineMemOperand *MMO = MF.getMachineMemOperand(
25511 MMO: TupleMemSD->getMemOperand(), Offset, Size: MemoryLocation::UnknownSize);
25512
25513 SDVTList VTs = DAG.getVTList(VTs: {VT, MVT::Other});
25514 SDValue Result = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs,
25515 Ops, MemVT, MMO);
25516 DAG.ReplaceAllUsesOfValueWith(From: Tuple.getValue(R: 1), To: Result.getValue(R: 1));
25517 return Result.getValue(R: 0);
25518 }
25519 case RISCVISD::TUPLE_INSERT: {
25520 // tuple_insert tuple, undef, idx -> tuple
25521 if (N->getOperand(Num: 1).isUndef())
25522 return N->getOperand(Num: 0);
25523 break;
25524 }
25525 case RISCVISD::VMERGE_VL: {
25526 // vmerge_vl allones, x, y, passthru, vl -> vmv_v_v passthru, x, vl
25527 SDValue Mask = N->getOperand(Num: 0);
25528 SDValue True = N->getOperand(Num: 1);
25529 SDValue Passthru = N->getOperand(Num: 3);
25530 SDValue VL = N->getOperand(Num: 4);
25531
25532 // Fixed vectors are wrapped in scalable containers, unwrap them.
25533 using namespace SDPatternMatch;
25534 SDValue SubVec;
25535 if (sd_match(N: Mask, P: m_InsertSubvector(Base: m_Undef(), Sub: m_Value(N&: SubVec), Idx: m_Zero())))
25536 Mask = SubVec;
25537
25538 if (!isOneOrOneSplat(V: Mask))
25539 break;
25540
25541 return DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
25542 N1: Passthru, N2: True, N3: VL);
25543 }
25544 case RISCVISD::VMV_V_V_VL: {
25545 // vmv_v_v passthru, splat(x), vl -> vmv_v_x passthru, x, vl
25546 SDValue Passthru = N->getOperand(Num: 0);
25547 SDValue Src = N->getOperand(Num: 1);
25548 SDValue VL = N->getOperand(Num: 2);
25549
25550 // Fixed vectors are wrapped in scalable containers, unwrap them.
25551 using namespace SDPatternMatch;
25552 SDValue SubVec;
25553 if (sd_match(N: Src, P: m_InsertSubvector(Base: m_Undef(), Sub: m_Value(N&: SubVec), Idx: m_Zero())))
25554 Src = SubVec;
25555
25556 SDValue SplatVal = DAG.getSplatValue(V: Src, /*LegalTypes=*/true);
25557 if (!SplatVal)
25558 break;
25559 MVT VT = N->getSimpleValueType(ResNo: 0);
25560 return lowerScalarSplat(Passthru, Scalar: SplatVal, VL, VT, DL: SDLoc(N), DAG,
25561 Subtarget);
25562 }
25563 case RISCVISD::VSLIDEDOWN_VL:
25564 case RISCVISD::VSLIDEUP_VL:
25565 if (N->getOperand(Num: 1)->isUndef())
25566 return N->getOperand(Num: 0);
25567 break;
25568 case RISCVISD::VSLIDE1UP_VL:
25569 case RISCVISD::VFSLIDE1UP_VL: {
25570 using namespace SDPatternMatch;
25571 SDValue SrcVec;
25572 SDLoc DL(N);
25573 MVT VT = N->getSimpleValueType(ResNo: 0);
25574 // If the scalar we're sliding in was extracted from the first element of a
25575 // vector, we can use that vector as the passthru in a normal slideup of 1.
25576 // This saves us an extract_element instruction (i.e. vfmv.f.s, vmv.x.s).
25577 if (!N->getOperand(Num: 0).isUndef() ||
25578 !sd_match(N: N->getOperand(Num: 2),
25579 P: m_AnyOf(preds: m_ExtractElt(Vec: m_Value(N&: SrcVec), Idx: m_Zero()),
25580 preds: m_Node(Opcode: RISCVISD::VMV_X_S, preds: m_Value(N&: SrcVec)))))
25581 break;
25582
25583 MVT SrcVecVT = SrcVec.getSimpleValueType();
25584 if (SrcVecVT.getVectorElementType() != VT.getVectorElementType())
25585 break;
25586 // Adapt the value type of source vector.
25587 if (SrcVecVT.isFixedLengthVector()) {
25588 SrcVecVT = getContainerForFixedLengthVector(VT: SrcVecVT);
25589 SrcVec = convertToScalableVector(VT: SrcVecVT, V: SrcVec, DAG, Subtarget);
25590 }
25591 if (SrcVecVT.getVectorMinNumElements() < VT.getVectorMinNumElements())
25592 SrcVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: SrcVec, Idx: 0);
25593 else
25594 SrcVec = DAG.getExtractSubvector(DL, VT, Vec: SrcVec, Idx: 0);
25595
25596 return getVSlideup(DAG, Subtarget, DL, VT, Passthru: SrcVec, Op: N->getOperand(Num: 1),
25597 Offset: DAG.getConstant(Val: 1, DL, VT: XLenVT), Mask: N->getOperand(Num: 3),
25598 VL: N->getOperand(Num: 4));
25599 }
25600 case ISD::MLOAD:
25601 return performMaskedLoadToVPLoadCombine(MLoad: cast<MaskedLoadSDNode>(Val: N), DAG);
25602 case ISD::VECTOR_INTERLEAVE:
25603 assert(Subtarget.hasStdExtZvzip());
25604 return performVECTOR_INTERLEAVECombine(N, DAG);
25605 }
25606
25607 return SDValue();
25608}
25609
25610bool RISCVTargetLowering::shouldTransformSignedTruncationCheck(
25611 EVT XVT, unsigned KeptBits) const {
25612 // For vectors, we don't have a preference..
25613 if (XVT.isVector())
25614 return false;
25615
25616 if (XVT != MVT::i32 && XVT != MVT::i64)
25617 return false;
25618
25619 // We can use sext.w for RV64 or an srai 31 on RV32.
25620 if (KeptBits == 32 || KeptBits == 64)
25621 return true;
25622
25623 // With Zbb we can use sext.h/sext.b.
25624 return Subtarget.hasStdExtZbb() &&
25625 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.is64Bit()) ||
25626 KeptBits == 16);
25627}
25628
25629bool RISCVTargetLowering::isDesirableToCommuteWithShift(
25630 const SDNode *N, CombineLevel Level) const {
25631 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
25632 N->getOpcode() == ISD::SRL) &&
25633 "Expected shift op");
25634
25635 // The following folds are only desirable if `(OP _, c1 << c2)` can be
25636 // materialised in fewer instructions than `(OP _, c1)`:
25637 //
25638 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
25639 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
25640 SDValue N0 = N->getOperand(Num: 0);
25641 EVT Ty = N0.getValueType();
25642
25643 // LD/ST will optimize constant Offset extraction, so when AddNode is used by
25644 // LD/ST, it can still complete the folding optimization operation performed
25645 // above.
25646 auto isUsedByLdSt = [](const SDNode *X, const SDNode *User) {
25647 for (SDNode *Use : X->users()) {
25648 // This use is the one we're on right now. Skip it
25649 if (Use == User || Use->getOpcode() == ISD::SELECT)
25650 continue;
25651 if (!isa<StoreSDNode>(Val: Use) && !isa<LoadSDNode>(Val: Use))
25652 return false;
25653 }
25654 return true;
25655 };
25656
25657 if (Ty.isScalarInteger() &&
25658 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
25659 if (N0.getOpcode() == ISD::ADD && !N0->hasOneUse())
25660 return isUsedByLdSt(N0.getNode(), N);
25661
25662 auto *C1 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
25663 auto *C2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
25664
25665 // Bail if we might break a sh{1,2,3}add/qc.shladd pattern.
25666 if (C2 && Subtarget.hasShlAdd(ShAmt: C2->getZExtValue()) && N->hasOneUse() &&
25667 N->user_begin()->getOpcode() == ISD::ADD &&
25668 !isUsedByLdSt(*N->user_begin(), nullptr) &&
25669 !isa<ConstantSDNode>(Val: N->user_begin()->getOperand(Num: 1)))
25670 return false;
25671
25672 if (C1 && C2) {
25673 const APInt &C1Int = C1->getAPIntValue();
25674 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
25675
25676 // We can materialise `c1 << c2` into an add immediate, so it's "free",
25677 // and the combine should happen, to potentially allow further combines
25678 // later.
25679 if (ShiftedC1Int.getSignificantBits() <= 64 &&
25680 isLegalAddImmediate(Imm: ShiftedC1Int.getSExtValue()))
25681 return true;
25682
25683 // We can materialise `c1` in an add immediate, so it's "free", and the
25684 // combine should be prevented.
25685 if (C1Int.getSignificantBits() <= 64 &&
25686 isLegalAddImmediate(Imm: C1Int.getSExtValue()))
25687 return false;
25688
25689 // Neither constant will fit into an immediate, so find materialisation
25690 // costs.
25691 int C1Cost =
25692 RISCVMatInt::getIntMatCost(Val: C1Int, Size: Ty.getSizeInBits(), STI: Subtarget,
25693 /*CompressionCost*/ true);
25694 int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
25695 Val: ShiftedC1Int, Size: Ty.getSizeInBits(), STI: Subtarget,
25696 /*CompressionCost*/ true);
25697
25698 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
25699 // combine should be prevented.
25700 if (C1Cost < ShiftedC1Cost)
25701 return false;
25702 }
25703 }
25704
25705 if (!N0->hasOneUse())
25706 return false;
25707
25708 if (N0->getOpcode() == ISD::SIGN_EXTEND &&
25709 N0->getOperand(Num: 0)->getOpcode() == ISD::ADD &&
25710 !N0->getOperand(Num: 0)->hasOneUse())
25711 return isUsedByLdSt(N0->getOperand(Num: 0).getNode(), N0.getNode());
25712
25713 return true;
25714}
25715
25716bool RISCVTargetLowering::targetShrinkDemandedConstant(
25717 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
25718 TargetLoweringOpt &TLO) const {
25719 // Delay this optimization as late as possible.
25720 if (!TLO.LegalOps)
25721 return false;
25722
25723 EVT VT = Op.getValueType();
25724 if (VT.isVector())
25725 return false;
25726
25727 unsigned Opcode = Op.getOpcode();
25728 if (Opcode != ISD::AND && Opcode != ISD::OR && Opcode != ISD::XOR)
25729 return false;
25730
25731 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
25732 if (!C)
25733 return false;
25734
25735 const APInt &Mask = C->getAPIntValue();
25736
25737 // Clear all non-demanded bits initially.
25738 APInt ShrunkMask = Mask & DemandedBits;
25739
25740 // Try to make a smaller immediate by setting undemanded bits.
25741
25742 APInt ExpandedMask = Mask | ~DemandedBits;
25743
25744 auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
25745 return ShrunkMask.isSubsetOf(RHS: Mask) && Mask.isSubsetOf(RHS: ExpandedMask);
25746 };
25747 auto UseMask = [Mask, Op, &TLO](const APInt &NewMask) -> bool {
25748 if (NewMask == Mask)
25749 return true;
25750 SDLoc DL(Op);
25751 SDValue NewC = TLO.DAG.getConstant(Val: NewMask, DL, VT: Op.getValueType());
25752 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
25753 N1: Op.getOperand(i: 0), N2: NewC);
25754 return TLO.CombineTo(O: Op, N: NewOp);
25755 };
25756
25757 // If the shrunk mask fits in sign extended 12 bits, let the target
25758 // independent code apply it.
25759 if (ShrunkMask.isSignedIntN(N: 12))
25760 return false;
25761
25762 // And has a few special cases for zext.
25763 if (Opcode == ISD::AND) {
25764 // Preserve (and X, 0xffff), if zext.h exists use zext.h,
25765 // otherwise use SLLI + SRLI.
25766 APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
25767 if (IsLegalMask(NewMask))
25768 return UseMask(NewMask);
25769
25770 // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
25771 if (VT == MVT::i64) {
25772 APInt NewMask = APInt(64, 0xffffffff);
25773 if (IsLegalMask(NewMask))
25774 return UseMask(NewMask);
25775 }
25776 }
25777
25778 // For the remaining optimizations, we need to be able to make a negative
25779 // number through a combination of mask and undemanded bits.
25780 if (ExpandedMask.isNegative()) {
25781 // What is the fewest number of bits we need to represent the negative
25782 // number.
25783 unsigned MinSignedBits = ExpandedMask.getSignificantBits();
25784
25785 // Try to make a 12 bit negative immediate. If that fails try to make a 32
25786 // bit negative immediate unless the shrunk immediate already fits in 32
25787 // bits. If we can't create a simm12, we shouldn't change opaque constants.
25788 if (MinSignedBits <= 12) {
25789 APInt NewMask = ShrunkMask;
25790 NewMask.setBitsFrom(11);
25791 assert(IsLegalMask(NewMask));
25792 return UseMask(NewMask);
25793 }
25794 if (!C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(N: 32)) {
25795 APInt NewMask = ShrunkMask;
25796 NewMask.setBitsFrom(31);
25797 assert(IsLegalMask(NewMask));
25798 return UseMask(NewMask);
25799 }
25800 }
25801
25802 // Try to form a constant that can be materialized with:
25803 // lui a0, hi20
25804 // addi(w) a0, a0, lo12
25805 // slli a1, a0, 32
25806 // add a0, a0, a1
25807 //
25808 // Or:
25809 // lui a0, hi20
25810 // addi(w) a0, a0, lo12
25811 // pack a0, a0, a0
25812 //
25813 if (!ShrunkMask.isSignedIntN(N: 32) && !C->isOpaque() && Opcode == ISD::AND &&
25814 VT == MVT::i64 && Subtarget.is64Bit()) {
25815 uint32_t Lo32Shrunk = Lo_32(Value: ShrunkMask.getZExtValue());
25816 uint32_t Hi32Shrunk = Hi_32(Value: ShrunkMask.getZExtValue());
25817
25818 // Only use this pattern if some bits in the upper and lower half must be
25819 // non-zero.
25820 if (Lo32Shrunk != Hi32Shrunk && Lo32Shrunk != 0 && Hi32Shrunk != 0) {
25821 // Find a 32-bit value that works for both halves.
25822 uint32_t Lo32Required = Lo32Shrunk | Hi32Shrunk;
25823
25824 // Replicate the 32-bit value to both halves.
25825 uint64_t DupConstant = Make_64(High: Lo32Required, Low: Lo32Required);
25826
25827 // Verify the new constant is legal.
25828 APInt CandidateMask(64, DupConstant);
25829 if (IsLegalMask(CandidateMask)) {
25830 unsigned OrigCost =
25831 RISCVMatInt::generateInstSeq(Val: ShrunkMask.getSExtValue(), STI: Subtarget)
25832 .size();
25833 unsigned NewCost =
25834 RISCVMatInt::generateInstSeq(Val: DupConstant, STI: Subtarget).size();
25835 // If the new sequence is shorter than the old sequence and won't
25836 // use a constant pool, make the change.
25837 if (NewCost < OrigCost && (!Subtarget.useConstantPoolForLargeInts() ||
25838 NewCost <= Subtarget.getMaxBuildIntsCost()))
25839 return UseMask(CandidateMask);
25840
25841 // For the 2 register form, if we're optimizing for size, only do
25842 // this if the original constant wasn't going to use a constant pool.
25843 if (!TLO.DAG.shouldOptForSize() ||
25844 !Subtarget.useConstantPoolForLargeInts() ||
25845 OrigCost <= Subtarget.getMaxBuildIntsCost()) {
25846 unsigned ShiftAmt, AddOpc;
25847 RISCVMatInt::InstSeq SeqLo = RISCVMatInt::generateTwoRegInstSeq(
25848 Val: DupConstant, STI: Subtarget, ShiftAmt, AddOpc);
25849 if (!SeqLo.empty()) {
25850 NewCost = SeqLo.size() + 2;
25851 if (NewCost < OrigCost &&
25852 (!Subtarget.useConstantPoolForLargeInts() ||
25853 (NewCost <= Subtarget.getMaxBuildIntsCost())))
25854 return UseMask(CandidateMask);
25855 }
25856 }
25857 }
25858 }
25859 }
25860
25861 return false;
25862}
25863
25864static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC) {
25865 static const uint64_t GREVMasks[] = {
25866 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
25867 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
25868
25869 for (unsigned Stage = 0; Stage != 6; ++Stage) {
25870 unsigned Shift = 1 << Stage;
25871 if (ShAmt & Shift) {
25872 uint64_t Mask = GREVMasks[Stage];
25873 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
25874 if (IsGORC)
25875 Res |= x;
25876 x = Res;
25877 }
25878 }
25879
25880 return x;
25881}
25882
25883void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
25884 KnownBits &Known,
25885 const APInt &DemandedElts,
25886 const SelectionDAG &DAG,
25887 unsigned Depth) const {
25888 unsigned BitWidth = Known.getBitWidth();
25889 unsigned Opc = Op.getOpcode();
25890 assert((Opc >= ISD::BUILTIN_OP_END ||
25891 Opc == ISD::INTRINSIC_WO_CHAIN ||
25892 Opc == ISD::INTRINSIC_W_CHAIN ||
25893 Opc == ISD::INTRINSIC_VOID) &&
25894 "Should use MaskedValueIsZero if you don't know whether Op"
25895 " is a target node!");
25896
25897 Known.resetAll();
25898 switch (Opc) {
25899 default: break;
25900 case RISCVISD::SELECT_CC: {
25901 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 4), Depth: Depth + 1);
25902 // If we don't know any bits, early out.
25903 if (Known.isUnknown())
25904 break;
25905 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 3), Depth: Depth + 1);
25906
25907 // Only known if known in both the LHS and RHS.
25908 Known = Known.intersectWith(RHS: Known2);
25909 break;
25910 }
25911 case RISCVISD::VCPOP_VL: {
25912 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
25913 Known.Zero.setBitsFrom(Known2.countMaxActiveBits());
25914 break;
25915 }
25916 case RISCVISD::CZERO_EQZ:
25917 case RISCVISD::CZERO_NEZ:
25918 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
25919 // Result is either all zero or operand 0. We can propagate zeros, but not
25920 // ones.
25921 Known.One.clearAllBits();
25922 break;
25923 case RISCVISD::REMUW: {
25924 KnownBits Known2;
25925 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
25926 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
25927 // We only care about the lower 32 bits.
25928 Known = KnownBits::urem(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 32));
25929 // Restore the original width by sign extending.
25930 Known = Known.sext(BitWidth);
25931 break;
25932 }
25933 case RISCVISD::DIVUW: {
25934 KnownBits Known2;
25935 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
25936 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
25937 // We only care about the lower 32 bits.
25938 Known = KnownBits::udiv(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 32));
25939 // Restore the original width by sign extending.
25940 Known = Known.sext(BitWidth);
25941 break;
25942 }
25943 case RISCVISD::SLLW: {
25944 KnownBits Known2;
25945 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
25946 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
25947 Known = KnownBits::shl(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
25948 // Restore the original width by sign extending.
25949 Known = Known.sext(BitWidth);
25950 break;
25951 }
25952 case RISCVISD::SRLW: {
25953 KnownBits Known2;
25954 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
25955 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
25956 Known = KnownBits::lshr(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
25957 // Restore the original width by sign extending.
25958 Known = Known.sext(BitWidth);
25959 break;
25960 }
25961 case RISCVISD::SRAW: {
25962 KnownBits Known2;
25963 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
25964 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
25965 Known = KnownBits::ashr(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
25966 // Restore the original width by sign extending.
25967 Known = Known.sext(BitWidth);
25968 break;
25969 }
25970 case RISCVISD::SHL_ADD: {
25971 KnownBits Known2;
25972 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
25973 unsigned ShAmt = Op.getConstantOperandVal(i: 1);
25974 Known <<= ShAmt;
25975 Known.Zero.setLowBits(ShAmt); // the <<= operator left these bits unknown
25976 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
25977 Known = KnownBits::add(LHS: Known, RHS: Known2);
25978 break;
25979 }
25980 case RISCVISD::CTZW: {
25981 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
25982 unsigned PossibleTZ = Known2.trunc(BitWidth: 32).countMaxTrailingZeros();
25983 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
25984 Known.Zero.setBitsFrom(LowBits);
25985 break;
25986 }
25987 case RISCVISD::CLZW: {
25988 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
25989 unsigned PossibleLZ = Known2.trunc(BitWidth: 32).countMaxLeadingZeros();
25990 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
25991 Known.Zero.setBitsFrom(LowBits);
25992 break;
25993 }
25994 case RISCVISD::CLSW: {
25995 // The upper 32 bits are ignored by the instruction, but ComputeNumSignBits
25996 // doesn't give us a way to ignore them. If there are fewer than 33 sign
25997 // bits in the input consider it as having no redundant sign bits. Otherwise
25998 // the lower bound of the result is NumSignBits-33. The maximum value of the
25999 // the result is 31.
26000 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
26001 unsigned MinRedundantSignBits = NumSignBits < 33 ? 0 : NumSignBits - 33;
26002 // Create a ConstantRange [MinRedundantSignBits, 32) and convert it to
26003 // KnownBits.
26004 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
26005 APInt(BitWidth, 32));
26006 Known = Range.toKnownBits();
26007 break;
26008 }
26009 case RISCVISD::BREV8:
26010 case RISCVISD::ORC_B: {
26011 // FIXME: This is based on the non-ratified Zbp GREV and GORC where a
26012 // control value of 7 is equivalent to brev8 and orc.b.
26013 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
26014 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
26015 // To compute zeros for ORC_B, we need to invert the value and invert it
26016 // back after. This inverting is harmless for BREV8.
26017 Known.Zero =
26018 ~computeGREVOrGORC(x: ~Known.Zero.getZExtValue(), ShAmt: 7, IsGORC);
26019 Known.One = computeGREVOrGORC(x: Known.One.getZExtValue(), ShAmt: 7, IsGORC);
26020 break;
26021 }
26022 case RISCVISD::USATI: {
26023 unsigned Width = Op.getConstantOperandVal(i: 1);
26024 Known.Zero.setBitsFrom(Width);
26025 break;
26026 }
26027 case RISCVISD::READ_VLENB: {
26028 // We can use the minimum and maximum VLEN values to bound VLENB. We
26029 // know VLEN must be a power of two.
26030 const unsigned MinVLenB = Subtarget.getRealMinVLen() / 8;
26031 const unsigned MaxVLenB = Subtarget.getRealMaxVLen() / 8;
26032 assert(MinVLenB > 0 && "READ_VLENB without vector extension enabled?");
26033 Known.Zero.setLowBits(Log2_32(Value: MinVLenB));
26034 Known.Zero.setBitsFrom(Log2_32(Value: MaxVLenB)+1);
26035 if (MaxVLenB == MinVLenB)
26036 Known.One.setBit(Log2_32(Value: MinVLenB));
26037 break;
26038 }
26039 case RISCVISD::FCLASS: {
26040 // fclass will only set one of the low 10 bits.
26041 Known.Zero.setBitsFrom(10);
26042 break;
26043 }
26044 case ISD::INTRINSIC_W_CHAIN:
26045 case ISD::INTRINSIC_WO_CHAIN: {
26046 unsigned IntNo =
26047 Op.getConstantOperandVal(i: Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
26048 switch (IntNo) {
26049 default:
26050 // We can't do anything for most intrinsics.
26051 break;
26052 case Intrinsic::riscv_vsetvli:
26053 case Intrinsic::riscv_vsetvlimax: {
26054 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
26055 unsigned VSEW = Op.getConstantOperandVal(i: HasAVL + 1);
26056 RISCVVType::VLMUL VLMUL =
26057 static_cast<RISCVVType::VLMUL>(Op.getConstantOperandVal(i: HasAVL + 2));
26058 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
26059 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul: VLMUL);
26060 uint64_t MaxVL = Subtarget.getRealMaxVLen() / SEW;
26061 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
26062
26063 // Result of vsetvli must be not larger than AVL.
26064 if (HasAVL && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
26065 MaxVL = std::min(a: MaxVL, b: Op.getConstantOperandVal(i: 1));
26066
26067 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
26068 if (BitWidth > KnownZeroFirstBit)
26069 Known.Zero.setBitsFrom(KnownZeroFirstBit);
26070 break;
26071 }
26072 }
26073 break;
26074 }
26075 }
26076}
26077
26078void RISCVTargetLowering::computeKnownBitsForTargetInstr(
26079 GISelValueTracking &Analysis, Register R, KnownBits &Known,
26080 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
26081 unsigned Depth) const {
26082 Known.resetAll();
26083
26084 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
26085 switch (MI->getOpcode()) {
26086 default:
26087 return;
26088 case RISCV::G_BREV8: {
26089 Analysis.computeKnownBitsImpl(R: MI->getOperand(i: 1).getReg(), Known,
26090 DemandedElts, Depth: Depth + 1);
26091
26092 Known.Zero =
26093 ~computeGREVOrGORC(x: ~Known.Zero.getZExtValue(), ShAmt: 7, /*IsGORC=*/false);
26094 Known.One =
26095 computeGREVOrGORC(x: Known.One.getZExtValue(), ShAmt: 7, /*IsGORC=*/false);
26096 return;
26097 }
26098 }
26099}
26100
26101unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
26102 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
26103 unsigned Depth) const {
26104 switch (Op.getOpcode()) {
26105 default:
26106 break;
26107 case RISCVISD::SELECT_CC: {
26108 unsigned Tmp =
26109 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth + 1);
26110 if (Tmp == 1) return 1; // Early out.
26111 unsigned Tmp2 =
26112 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 4), DemandedElts, Depth: Depth + 1);
26113 return std::min(a: Tmp, b: Tmp2);
26114 }
26115 case RISCVISD::CZERO_EQZ:
26116 case RISCVISD::CZERO_NEZ:
26117 // Output is either all zero or operand 0. We can propagate sign bit count
26118 // from operand 0.
26119 return DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
26120 case RISCVISD::NEGW_MAX: {
26121 // We expand this at isel to negw+max. The result will have 33 sign bits
26122 // if the input has at least 33 sign bits.
26123 unsigned Tmp =
26124 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
26125 if (Tmp < 33) return 1;
26126 return 33;
26127 }
26128 case RISCVISD::SRAW: {
26129 unsigned Tmp =
26130 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
26131 // sraw produces at least 33 sign bits. If the input already has more than
26132 // 33 sign bits sraw, will preserve them.
26133 // TODO: A more precise answer could be calculated depending on known bits
26134 // in the shift amount.
26135 return std::max(a: Tmp, b: 33U);
26136 }
26137 case RISCVISD::SLLW:
26138 case RISCVISD::SRLW:
26139 case RISCVISD::DIVW:
26140 case RISCVISD::DIVUW:
26141 case RISCVISD::REMUW:
26142 case RISCVISD::ROLW:
26143 case RISCVISD::RORW:
26144 case RISCVISD::ABSW:
26145 case RISCVISD::FCVT_W_RV64:
26146 case RISCVISD::FCVT_WU_RV64:
26147 case RISCVISD::STRICT_FCVT_W_RV64:
26148 case RISCVISD::STRICT_FCVT_WU_RV64:
26149 // TODO: As the result is sign-extended, this is conservatively correct.
26150 return 33;
26151 case RISCVISD::SATI: {
26152 unsigned Width = Op.getConstantOperandVal(i: 1);
26153 return Op.getScalarValueSizeInBits() - Width;
26154 }
26155 case RISCVISD::VMV_X_S: {
26156 // The number of sign bits of the scalar result is computed by obtaining the
26157 // element type of the input vector operand, subtracting its width from the
26158 // XLEN, and then adding one (sign bit within the element type). If the
26159 // element type is wider than XLen, the least-significant XLEN bits are
26160 // taken.
26161 unsigned XLen = Subtarget.getXLen();
26162 unsigned EltBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
26163 if (EltBits <= XLen)
26164 return XLen - EltBits + 1;
26165 break;
26166 }
26167 case ISD::INTRINSIC_W_CHAIN: {
26168 unsigned IntNo = Op.getConstantOperandVal(i: 1);
26169 switch (IntNo) {
26170 default:
26171 break;
26172 case Intrinsic::riscv_masked_atomicrmw_xchg:
26173 case Intrinsic::riscv_masked_atomicrmw_add:
26174 case Intrinsic::riscv_masked_atomicrmw_sub:
26175 case Intrinsic::riscv_masked_atomicrmw_nand:
26176 case Intrinsic::riscv_masked_atomicrmw_max:
26177 case Intrinsic::riscv_masked_atomicrmw_min:
26178 case Intrinsic::riscv_masked_atomicrmw_umax:
26179 case Intrinsic::riscv_masked_atomicrmw_umin:
26180 case Intrinsic::riscv_masked_cmpxchg:
26181 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
26182 // narrow atomic operation. These are implemented using atomic
26183 // operations at the minimum supported atomicrmw/cmpxchg width whose
26184 // result is then sign extended to XLEN. With +A, the minimum width is
26185 // 32 for both 64 and 32.
26186 assert(getMinCmpXchgSizeInBits() == 32);
26187 assert(Subtarget.hasStdExtZalrsc());
26188 return Op.getValueSizeInBits() - 31;
26189 }
26190 break;
26191 }
26192 }
26193
26194 return 1;
26195}
26196
26197bool RISCVTargetLowering::SimplifyDemandedBitsForTargetNode(
26198 SDValue Op, const APInt &OriginalDemandedBits,
26199 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
26200 unsigned Depth) const {
26201 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
26202
26203 switch (Op.getOpcode()) {
26204 case RISCVISD::BREV8:
26205 case RISCVISD::ORC_B: {
26206 KnownBits Known2;
26207 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
26208 // For BREV8, we need to do BREV8 on the demanded bits.
26209 // For ORC_B, any bit in the output demandeds all bits from the same byte.
26210 // So we need to do ORC_B on the demanded bits.
26211 APInt DemandedBits =
26212 APInt(BitWidth, computeGREVOrGORC(x: OriginalDemandedBits.getZExtValue(),
26213 ShAmt: 7, IsGORC));
26214 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits,
26215 DemandedElts: OriginalDemandedElts, Known&: Known2, TLO, Depth: Depth + 1))
26216 return true;
26217
26218 // To compute zeros for ORC_B, we need to invert the value and invert it
26219 // back after. This inverting is harmless for BREV8.
26220 Known.Zero = ~computeGREVOrGORC(x: ~Known2.Zero.getZExtValue(), ShAmt: 7, IsGORC);
26221 Known.One = computeGREVOrGORC(x: Known2.One.getZExtValue(), ShAmt: 7, IsGORC);
26222 return false;
26223 }
26224 }
26225
26226 return TargetLowering::SimplifyDemandedBitsForTargetNode(
26227 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
26228}
26229
26230bool RISCVTargetLowering::canCreateUndefOrPoisonForTargetNode(
26231 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
26232 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
26233
26234 // TODO: Add more target nodes.
26235 switch (Op.getOpcode()) {
26236 case RISCVISD::READ_VLENB:
26237 return false;
26238 case RISCVISD::SLLW:
26239 case RISCVISD::SRAW:
26240 case RISCVISD::SRLW:
26241 case RISCVISD::RORW:
26242 case RISCVISD::ROLW:
26243 // Only the lower 5 bits of RHS are read, guaranteeing the rotate/shift
26244 // amount is bounds.
26245 return false;
26246 case RISCVISD::SELECT_CC:
26247 // Integer comparisons cannot create poison.
26248 assert(Op.getOperand(0).getValueType().isInteger() &&
26249 "RISCVISD::SELECT_CC only compares integers");
26250 return false;
26251 }
26252 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
26253 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
26254}
26255
26256const Constant *
26257RISCVTargetLowering::getTargetConstantFromLoad(LoadSDNode *Ld) const {
26258 assert(Ld && "Unexpected null LoadSDNode");
26259 if (!ISD::isNormalLoad(N: Ld))
26260 return nullptr;
26261
26262 SDValue Ptr = Ld->getBasePtr();
26263
26264 // Only constant pools with no offset are supported.
26265 auto GetSupportedConstantPool = [](SDValue Ptr) -> ConstantPoolSDNode * {
26266 auto *CNode = dyn_cast<ConstantPoolSDNode>(Val&: Ptr);
26267 if (!CNode || CNode->isMachineConstantPoolEntry() ||
26268 CNode->getOffset() != 0)
26269 return nullptr;
26270
26271 return CNode;
26272 };
26273
26274 // Simple case, LLA.
26275 if (Ptr.getOpcode() == RISCVISD::LLA) {
26276 auto *CNode = GetSupportedConstantPool(Ptr.getOperand(i: 0));
26277 if (!CNode || CNode->getTargetFlags() != 0)
26278 return nullptr;
26279
26280 return CNode->getConstVal();
26281 }
26282
26283 // Look for a HI and ADD_LO pair.
26284 if (Ptr.getOpcode() != RISCVISD::ADD_LO ||
26285 Ptr.getOperand(i: 0).getOpcode() != RISCVISD::HI)
26286 return nullptr;
26287
26288 auto *CNodeLo = GetSupportedConstantPool(Ptr.getOperand(i: 1));
26289 auto *CNodeHi = GetSupportedConstantPool(Ptr.getOperand(i: 0).getOperand(i: 0));
26290
26291 if (!CNodeLo || CNodeLo->getTargetFlags() != RISCVII::MO_LO ||
26292 !CNodeHi || CNodeHi->getTargetFlags() != RISCVII::MO_HI)
26293 return nullptr;
26294
26295 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
26296 return nullptr;
26297
26298 return CNodeLo->getConstVal();
26299}
26300
26301static MachineBasicBlock *emitReadCounterWidePseudo(MachineInstr &MI,
26302 MachineBasicBlock *BB) {
26303 assert(MI.getOpcode() == RISCV::ReadCounterWide && "Unexpected instruction");
26304
26305 // To read a 64-bit counter CSR on a 32-bit target, we read the two halves.
26306 // Should the count have wrapped while it was being read, we need to try
26307 // again.
26308 // For example:
26309 // ```
26310 // read:
26311 // csrrs x3, counterh # load high word of counter
26312 // csrrs x2, counter # load low word of counter
26313 // csrrs x4, counterh # load high word of counter
26314 // bne x3, x4, read # check if high word reads match, otherwise try again
26315 // ```
26316
26317 MachineFunction &MF = *BB->getParent();
26318 const BasicBlock *LLVMBB = BB->getBasicBlock();
26319 MachineFunction::iterator It = ++BB->getIterator();
26320
26321 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
26322 MF.insert(MBBI: It, MBB: LoopMBB);
26323
26324 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
26325 MF.insert(MBBI: It, MBB: DoneMBB);
26326
26327 // Transfer the remainder of BB and its successor edges to DoneMBB.
26328 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
26329 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
26330 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
26331
26332 BB->addSuccessor(Succ: LoopMBB);
26333
26334 MachineRegisterInfo &RegInfo = MF.getRegInfo();
26335 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
26336 Register LoReg = MI.getOperand(i: 0).getReg();
26337 Register HiReg = MI.getOperand(i: 1).getReg();
26338 int64_t LoCounter = MI.getOperand(i: 2).getImm();
26339 int64_t HiCounter = MI.getOperand(i: 3).getImm();
26340 DebugLoc DL = MI.getDebugLoc();
26341
26342 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
26343 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
26344 .addImm(Val: HiCounter)
26345 .addReg(RegNo: RISCV::X0);
26346 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
26347 .addImm(Val: LoCounter)
26348 .addReg(RegNo: RISCV::X0);
26349 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
26350 .addImm(Val: HiCounter)
26351 .addReg(RegNo: RISCV::X0);
26352
26353 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
26354 .addReg(RegNo: HiReg)
26355 .addReg(RegNo: ReadAgainReg)
26356 .addMBB(MBB: LoopMBB);
26357
26358 LoopMBB->addSuccessor(Succ: LoopMBB);
26359 LoopMBB->addSuccessor(Succ: DoneMBB);
26360
26361 MI.eraseFromParent();
26362
26363 return DoneMBB;
26364}
26365
26366static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
26367 MachineBasicBlock *BB,
26368 const RISCVSubtarget &Subtarget) {
26369 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
26370
26371 MachineFunction &MF = *BB->getParent();
26372 DebugLoc DL = MI.getDebugLoc();
26373 const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
26374 Register LoReg = MI.getOperand(i: 0).getReg();
26375 Register HiReg = MI.getOperand(i: 1).getReg();
26376 Register SrcReg = MI.getOperand(i: 2).getReg();
26377
26378 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
26379 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
26380
26381 TII.storeRegToStackSlot(MBB&: *BB, MBBI: MI, SrcReg, IsKill: MI.getOperand(i: 2).isKill(), FrameIndex: FI, RC: SrcRC,
26382 VReg: Register());
26383 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
26384 MachineMemOperand *MMOLo =
26385 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(8));
26386 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
26387 PtrInfo: MPI.getWithOffset(O: 4), F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(8));
26388
26389 // For big-endian, the high part is at offset 0 and the low part at offset 4.
26390 if (!Subtarget.isLittleEndian())
26391 std::swap(a&: LoReg, b&: HiReg);
26392
26393 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::LW), DestReg: LoReg)
26394 .addFrameIndex(Idx: FI)
26395 .addImm(Val: 0)
26396 .addMemOperand(MMO: MMOLo);
26397 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::LW), DestReg: HiReg)
26398 .addFrameIndex(Idx: FI)
26399 .addImm(Val: 4)
26400 .addMemOperand(MMO: MMOHi);
26401 MI.eraseFromParent(); // The pseudo instruction is gone now.
26402 return BB;
26403}
26404
26405static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
26406 MachineBasicBlock *BB,
26407 const RISCVSubtarget &Subtarget) {
26408 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
26409 "Unexpected instruction");
26410
26411 MachineFunction &MF = *BB->getParent();
26412 DebugLoc DL = MI.getDebugLoc();
26413 const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
26414 Register DstReg = MI.getOperand(i: 0).getReg();
26415 Register LoReg = MI.getOperand(i: 1).getReg();
26416 Register HiReg = MI.getOperand(i: 2).getReg();
26417 bool KillLo = MI.getOperand(i: 1).isKill();
26418 bool KillHi = MI.getOperand(i: 2).isKill();
26419
26420 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
26421 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
26422
26423 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
26424 MachineMemOperand *MMOLo =
26425 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(8));
26426 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
26427 PtrInfo: MPI.getWithOffset(O: 4), F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(8));
26428
26429 // For big-endian, store the high part at offset 0 and the low part at
26430 // offset 4.
26431 if (!Subtarget.isLittleEndian()) {
26432 std::swap(a&: LoReg, b&: HiReg);
26433 std::swap(a&: KillLo, b&: KillHi);
26434 }
26435
26436 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::SW))
26437 .addReg(RegNo: LoReg, Flags: getKillRegState(B: KillLo))
26438 .addFrameIndex(Idx: FI)
26439 .addImm(Val: 0)
26440 .addMemOperand(MMO: MMOLo);
26441 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::SW))
26442 .addReg(RegNo: HiReg, Flags: getKillRegState(B: KillHi))
26443 .addFrameIndex(Idx: FI)
26444 .addImm(Val: 4)
26445 .addMemOperand(MMO: MMOHi);
26446 TII.loadRegFromStackSlot(MBB&: *BB, MBBI: MI, DstReg, FrameIndex: FI, RC: DstRC, VReg: Register());
26447 MI.eraseFromParent(); // The pseudo instruction is gone now.
26448 return BB;
26449}
26450
26451static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
26452 unsigned RelOpcode, unsigned EqOpcode,
26453 const RISCVSubtarget &Subtarget) {
26454 DebugLoc DL = MI.getDebugLoc();
26455 Register DstReg = MI.getOperand(i: 0).getReg();
26456 Register Src1Reg = MI.getOperand(i: 1).getReg();
26457 Register Src2Reg = MI.getOperand(i: 2).getReg();
26458 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
26459 Register SavedFFlags = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
26460 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
26461
26462 // Save the current FFLAGS.
26463 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::ReadFFLAGS), DestReg: SavedFFlags);
26464
26465 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RelOpcode), DestReg: DstReg)
26466 .addReg(RegNo: Src1Reg)
26467 .addReg(RegNo: Src2Reg);
26468 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
26469 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
26470
26471 // Restore the FFLAGS.
26472 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::WriteFFLAGS))
26473 .addReg(RegNo: SavedFFlags, Flags: RegState::Kill);
26474
26475 // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
26476 auto MIB2 = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: EqOpcode), DestReg: RISCV::X0)
26477 .addReg(RegNo: Src1Reg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()))
26478 .addReg(RegNo: Src2Reg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
26479 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
26480 MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
26481
26482 // Erase the pseudoinstruction.
26483 MI.eraseFromParent();
26484 return BB;
26485}
26486
26487static MachineBasicBlock *
26488EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second,
26489 MachineBasicBlock *ThisMBB,
26490 const RISCVSubtarget &Subtarget) {
26491 // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5)
26492 // Without this, custom-inserter would have generated:
26493 //
26494 // A
26495 // | \
26496 // | B
26497 // | /
26498 // C
26499 // | \
26500 // | D
26501 // | /
26502 // E
26503 //
26504 // A: X = ...; Y = ...
26505 // B: empty
26506 // C: Z = PHI [X, A], [Y, B]
26507 // D: empty
26508 // E: PHI [X, C], [Z, D]
26509 //
26510 // If we lower both Select_FPRX_ in a single step, we can instead generate:
26511 //
26512 // A
26513 // | \
26514 // | C
26515 // | /|
26516 // |/ |
26517 // | |
26518 // | D
26519 // | /
26520 // E
26521 //
26522 // A: X = ...; Y = ...
26523 // D: empty
26524 // E: PHI [X, A], [X, C], [Y, D]
26525
26526 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
26527 const DebugLoc &DL = First.getDebugLoc();
26528 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
26529 MachineFunction *F = ThisMBB->getParent();
26530 MachineBasicBlock *FirstMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
26531 MachineBasicBlock *SecondMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
26532 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
26533 MachineFunction::iterator It = ++ThisMBB->getIterator();
26534 F->insert(MBBI: It, MBB: FirstMBB);
26535 F->insert(MBBI: It, MBB: SecondMBB);
26536 F->insert(MBBI: It, MBB: SinkMBB);
26537
26538 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
26539 SinkMBB->splice(Where: SinkMBB->begin(), Other: ThisMBB,
26540 From: std::next(x: MachineBasicBlock::iterator(First)),
26541 To: ThisMBB->end());
26542 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: ThisMBB);
26543
26544 // Fallthrough block for ThisMBB.
26545 ThisMBB->addSuccessor(Succ: FirstMBB);
26546 // Fallthrough block for FirstMBB.
26547 FirstMBB->addSuccessor(Succ: SecondMBB);
26548 ThisMBB->addSuccessor(Succ: SinkMBB);
26549 FirstMBB->addSuccessor(Succ: SinkMBB);
26550 // This is fallthrough.
26551 SecondMBB->addSuccessor(Succ: SinkMBB);
26552
26553 auto FirstCC = static_cast<RISCVCC::CondCode>(First.getOperand(i: 3).getImm());
26554 Register FLHS = First.getOperand(i: 1).getReg();
26555 Register FRHS = First.getOperand(i: 2).getReg();
26556 // Insert appropriate branch.
26557 BuildMI(BB: FirstMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC: FirstCC, SelectOpc: First.getOpcode())))
26558 .addReg(RegNo: FLHS)
26559 .addReg(RegNo: FRHS)
26560 .addMBB(MBB: SinkMBB);
26561
26562 Register SLHS = Second.getOperand(i: 1).getReg();
26563 Register SRHS = Second.getOperand(i: 2).getReg();
26564 Register Op1Reg4 = First.getOperand(i: 4).getReg();
26565 Register Op1Reg5 = First.getOperand(i: 5).getReg();
26566
26567 auto SecondCC = static_cast<RISCVCC::CondCode>(Second.getOperand(i: 3).getImm());
26568 // Insert appropriate branch.
26569 BuildMI(BB: ThisMBB, MIMD: DL,
26570 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC: SecondCC, SelectOpc: Second.getOpcode())))
26571 .addReg(RegNo: SLHS)
26572 .addReg(RegNo: SRHS)
26573 .addMBB(MBB: SinkMBB);
26574
26575 Register DestReg = Second.getOperand(i: 0).getReg();
26576 Register Op2Reg4 = Second.getOperand(i: 4).getReg();
26577 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: RISCV::PHI), DestReg)
26578 .addReg(RegNo: Op2Reg4)
26579 .addMBB(MBB: ThisMBB)
26580 .addReg(RegNo: Op1Reg4)
26581 .addMBB(MBB: FirstMBB)
26582 .addReg(RegNo: Op1Reg5)
26583 .addMBB(MBB: SecondMBB);
26584
26585 // Now remove the Select_FPRX_s.
26586 First.eraseFromParent();
26587 Second.eraseFromParent();
26588 return SinkMBB;
26589}
26590
26591static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
26592 MachineBasicBlock *BB,
26593 const RISCVSubtarget &Subtarget) {
26594 // To "insert" Select_* instructions, we actually have to insert the triangle
26595 // control-flow pattern. The incoming instructions know the destination vreg
26596 // to set, the condition code register to branch on, the true/false values to
26597 // select between, and the condcode to use to select the appropriate branch.
26598 //
26599 // We produce the following control flow:
26600 // HeadMBB
26601 // | \
26602 // | IfFalseMBB
26603 // | /
26604 // TailMBB
26605 //
26606 // When we find a sequence of selects we attempt to optimize their emission
26607 // by sharing the control flow. Currently we only handle cases where we have
26608 // multiple selects with the exact same condition (same LHS, RHS and CC).
26609 // The selects may be interleaved with other instructions if the other
26610 // instructions meet some requirements we deem safe:
26611 // - They are not pseudo instructions.
26612 // - They are debug instructions. Otherwise,
26613 // - They do not have side-effects, do not access memory and their inputs do
26614 // not depend on the results of the select pseudo-instructions.
26615 // - They don't adjust stack.
26616 // The TrueV/FalseV operands of the selects cannot depend on the result of
26617 // previous selects in the sequence.
26618 // These conditions could be further relaxed. See the X86 target for a
26619 // related approach and more information.
26620 //
26621 // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5))
26622 // is checked here and handled by a separate function -
26623 // EmitLoweredCascadedSelect.
26624
26625 auto Next = next_nodbg(It: MI.getIterator(), End: BB->instr_end());
26626 if (MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
26627 MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
26628 Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
26629 Next->getOperand(i: 5).getReg() == MI.getOperand(i: 0).getReg() &&
26630 Next->getOperand(i: 5).isKill())
26631 return EmitLoweredCascadedSelect(First&: MI, Second&: *Next, ThisMBB: BB, Subtarget);
26632
26633 Register LHS = MI.getOperand(i: 1).getReg();
26634 Register RHS;
26635 if (MI.getOperand(i: 2).isReg())
26636 RHS = MI.getOperand(i: 2).getReg();
26637 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
26638
26639 SmallVector<MachineInstr *, 4> SelectDebugValues;
26640 SmallSet<Register, 4> SelectDests;
26641 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
26642
26643 MachineInstr *LastSelectPseudo = &MI;
26644 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
26645
26646 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
26647 SequenceMBBI != E; ++SequenceMBBI) {
26648 if (SequenceMBBI->isDebugInstr())
26649 continue;
26650 if (RISCVInstrInfo::isSelectPseudo(MI: *SequenceMBBI)) {
26651 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
26652 !SequenceMBBI->getOperand(i: 2).isReg() ||
26653 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
26654 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
26655 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
26656 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
26657 break;
26658 LastSelectPseudo = &*SequenceMBBI;
26659 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
26660 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
26661 continue;
26662 }
26663 if (SequenceMBBI->hasUnmodeledSideEffects() ||
26664 SequenceMBBI->mayLoadOrStore() ||
26665 SequenceMBBI->usesCustomInsertionHook() ||
26666 TII.isFrameInstr(I: *SequenceMBBI) ||
26667 SequenceMBBI->isStackAligningInlineAsm())
26668 break;
26669 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
26670 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
26671 }))
26672 break;
26673 }
26674
26675 const BasicBlock *LLVM_BB = BB->getBasicBlock();
26676 DebugLoc DL = MI.getDebugLoc();
26677 MachineFunction::iterator I = ++BB->getIterator();
26678
26679 MachineBasicBlock *HeadMBB = BB;
26680 MachineFunction *F = BB->getParent();
26681 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
26682 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
26683
26684 F->insert(MBBI: I, MBB: IfFalseMBB);
26685 F->insert(MBBI: I, MBB: TailMBB);
26686
26687 // Set the call frame size on entry to the new basic blocks.
26688 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
26689 IfFalseMBB->setCallFrameSize(CallFrameSize);
26690 TailMBB->setCallFrameSize(CallFrameSize);
26691
26692 // Transfer debug instructions associated with the selects to TailMBB.
26693 for (MachineInstr *DebugInstr : SelectDebugValues) {
26694 TailMBB->push_back(MI: DebugInstr->removeFromParent());
26695 }
26696
26697 // Move all instructions after the sequence to TailMBB.
26698 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
26699 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
26700 // Update machine-CFG edges by transferring all successors of the current
26701 // block to the new block which will contain the Phi nodes for the selects.
26702 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
26703 // Set the successors for HeadMBB.
26704 HeadMBB->addSuccessor(Succ: IfFalseMBB);
26705 HeadMBB->addSuccessor(Succ: TailMBB);
26706
26707 // Insert appropriate branch.
26708 if (MI.getOperand(i: 2).isImm())
26709 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC, SelectOpc: MI.getOpcode())))
26710 .addReg(RegNo: LHS)
26711 .addImm(Val: MI.getOperand(i: 2).getImm())
26712 .addMBB(MBB: TailMBB);
26713 else
26714 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC, SelectOpc: MI.getOpcode())))
26715 .addReg(RegNo: LHS)
26716 .addReg(RegNo: RHS)
26717 .addMBB(MBB: TailMBB);
26718
26719 // IfFalseMBB just falls through to TailMBB.
26720 IfFalseMBB->addSuccessor(Succ: TailMBB);
26721
26722 // Create PHIs for all of the select pseudo-instructions.
26723 auto SelectMBBI = MI.getIterator();
26724 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
26725 auto InsertionPoint = TailMBB->begin();
26726 while (SelectMBBI != SelectEnd) {
26727 auto Next = std::next(x: SelectMBBI);
26728 if (RISCVInstrInfo::isSelectPseudo(MI: *SelectMBBI)) {
26729 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
26730 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
26731 MCID: TII.get(Opcode: RISCV::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
26732 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
26733 .addMBB(MBB: HeadMBB)
26734 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
26735 .addMBB(MBB: IfFalseMBB);
26736 SelectMBBI->eraseFromParent();
26737 }
26738 SelectMBBI = Next;
26739 }
26740
26741 F->getProperties().resetNoPHIs();
26742 return TailMBB;
26743}
26744
26745// Helper to find Masked Pseudo instruction from MC instruction, LMUL and SEW.
26746static const RISCV::RISCVMaskedPseudoInfo *
26747lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVVType::VLMUL LMul, unsigned SEW) {
26748 const RISCVVInversePseudosTable::PseudoInfo *Inverse =
26749 RISCVVInversePseudosTable::getBaseInfo(BaseInstr: MCOpcode, VLMul: LMul, SEW);
26750 assert(Inverse && "Unexpected LMUL and SEW pair for instruction");
26751 const RISCV::RISCVMaskedPseudoInfo *Masked =
26752 RISCV::lookupMaskedIntrinsicByUnmasked(UnmaskedPseudo: Inverse->Pseudo);
26753 assert(Masked && "Could not find masked instruction for LMUL and SEW pair");
26754 return Masked;
26755}
26756
26757static MachineBasicBlock *emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI,
26758 MachineBasicBlock *BB,
26759 unsigned CVTXOpc) {
26760 DebugLoc DL = MI.getDebugLoc();
26761
26762 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
26763
26764 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
26765 Register SavedFFLAGS = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
26766
26767 // Save the old value of FFLAGS.
26768 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::ReadFFLAGS), DestReg: SavedFFLAGS);
26769
26770 assert(MI.getNumOperands() == 7);
26771
26772 // Emit a VFCVT_X_F
26773 const TargetRegisterInfo *TRI =
26774 BB->getParent()->getSubtarget().getRegisterInfo();
26775 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: 0, TII: &TII, TRI);
26776 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
26777 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: CVTXOpc), DestReg: Tmp)
26778 .add(MO: MI.getOperand(i: 1))
26779 .add(MO: MI.getOperand(i: 2))
26780 .add(MO: MI.getOperand(i: 3))
26781 .add(MO: MachineOperand::CreateImm(Val: 7)) // frm = DYN
26782 .add(MO: MI.getOperand(i: 4))
26783 .add(MO: MI.getOperand(i: 5))
26784 .add(MO: MI.getOperand(i: 6))
26785 .add(MO: MachineOperand::CreateReg(Reg: RISCV::FRM,
26786 /*IsDef*/ isDef: false,
26787 /*IsImp*/ isImp: true));
26788
26789 // Emit a VFCVT_F_X
26790 RISCVVType::VLMUL LMul = RISCVII::getLMul(TSFlags: MI.getDesc().TSFlags);
26791 unsigned Log2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
26792 // There is no E8 variant for VFCVT_F_X.
26793 assert(Log2SEW >= 4);
26794 unsigned CVTFOpc =
26795 lookupMaskedIntrinsic(MCOpcode: RISCV::VFCVT_F_X_V, LMul, SEW: 1 << Log2SEW)
26796 ->MaskedPseudo;
26797
26798 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: CVTFOpc))
26799 .add(MO: MI.getOperand(i: 0))
26800 .add(MO: MI.getOperand(i: 1))
26801 .addReg(RegNo: Tmp)
26802 .add(MO: MI.getOperand(i: 3))
26803 .add(MO: MachineOperand::CreateImm(Val: 7)) // frm = DYN
26804 .add(MO: MI.getOperand(i: 4))
26805 .add(MO: MI.getOperand(i: 5))
26806 .add(MO: MI.getOperand(i: 6))
26807 .add(MO: MachineOperand::CreateReg(Reg: RISCV::FRM,
26808 /*IsDef*/ isDef: false,
26809 /*IsImp*/ isImp: true));
26810
26811 // Restore FFLAGS.
26812 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::WriteFFLAGS))
26813 .addReg(RegNo: SavedFFLAGS, Flags: RegState::Kill);
26814
26815 // Erase the pseudoinstruction.
26816 MI.eraseFromParent();
26817 return BB;
26818}
26819
26820static MachineBasicBlock *emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB,
26821 const RISCVSubtarget &Subtarget) {
26822 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
26823 const TargetRegisterClass *RC;
26824 switch (MI.getOpcode()) {
26825 default:
26826 llvm_unreachable("Unexpected opcode");
26827 case RISCV::PseudoFROUND_H:
26828 CmpOpc = RISCV::FLT_H;
26829 F2IOpc = RISCV::FCVT_W_H;
26830 I2FOpc = RISCV::FCVT_H_W;
26831 FSGNJOpc = RISCV::FSGNJ_H;
26832 FSGNJXOpc = RISCV::FSGNJX_H;
26833 RC = &RISCV::FPR16RegClass;
26834 break;
26835 case RISCV::PseudoFROUND_H_INX:
26836 CmpOpc = RISCV::FLT_H_INX;
26837 F2IOpc = RISCV::FCVT_W_H_INX;
26838 I2FOpc = RISCV::FCVT_H_W_INX;
26839 FSGNJOpc = RISCV::FSGNJ_H_INX;
26840 FSGNJXOpc = RISCV::FSGNJX_H_INX;
26841 RC = &RISCV::GPRF16RegClass;
26842 break;
26843 case RISCV::PseudoFROUND_S:
26844 CmpOpc = RISCV::FLT_S;
26845 F2IOpc = RISCV::FCVT_W_S;
26846 I2FOpc = RISCV::FCVT_S_W;
26847 FSGNJOpc = RISCV::FSGNJ_S;
26848 FSGNJXOpc = RISCV::FSGNJX_S;
26849 RC = &RISCV::FPR32RegClass;
26850 break;
26851 case RISCV::PseudoFROUND_S_INX:
26852 CmpOpc = RISCV::FLT_S_INX;
26853 F2IOpc = RISCV::FCVT_W_S_INX;
26854 I2FOpc = RISCV::FCVT_S_W_INX;
26855 FSGNJOpc = RISCV::FSGNJ_S_INX;
26856 FSGNJXOpc = RISCV::FSGNJX_S_INX;
26857 RC = &RISCV::GPRF32RegClass;
26858 break;
26859 case RISCV::PseudoFROUND_D:
26860 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
26861 CmpOpc = RISCV::FLT_D;
26862 F2IOpc = RISCV::FCVT_L_D;
26863 I2FOpc = RISCV::FCVT_D_L;
26864 FSGNJOpc = RISCV::FSGNJ_D;
26865 FSGNJXOpc = RISCV::FSGNJX_D;
26866 RC = &RISCV::FPR64RegClass;
26867 break;
26868 case RISCV::PseudoFROUND_D_INX:
26869 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
26870 CmpOpc = RISCV::FLT_D_INX;
26871 F2IOpc = RISCV::FCVT_L_D_INX;
26872 I2FOpc = RISCV::FCVT_D_L_INX;
26873 FSGNJOpc = RISCV::FSGNJ_D_INX;
26874 FSGNJXOpc = RISCV::FSGNJX_D_INX;
26875 RC = &RISCV::GPRRegClass;
26876 break;
26877 }
26878
26879 const BasicBlock *BB = MBB->getBasicBlock();
26880 DebugLoc DL = MI.getDebugLoc();
26881 MachineFunction::iterator I = ++MBB->getIterator();
26882
26883 MachineFunction *F = MBB->getParent();
26884 MachineBasicBlock *CvtMBB = F->CreateMachineBasicBlock(BB);
26885 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB);
26886
26887 F->insert(MBBI: I, MBB: CvtMBB);
26888 F->insert(MBBI: I, MBB: DoneMBB);
26889 // Move all instructions after the sequence to DoneMBB.
26890 DoneMBB->splice(Where: DoneMBB->end(), Other: MBB, From: MachineBasicBlock::iterator(MI),
26891 To: MBB->end());
26892 // Update machine-CFG edges by transferring all successors of the current
26893 // block to the new block which will contain the Phi nodes for the selects.
26894 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
26895 // Set the successors for MBB.
26896 MBB->addSuccessor(Succ: CvtMBB);
26897 MBB->addSuccessor(Succ: DoneMBB);
26898
26899 Register DstReg = MI.getOperand(i: 0).getReg();
26900 Register SrcReg = MI.getOperand(i: 1).getReg();
26901 Register MaxReg = MI.getOperand(i: 2).getReg();
26902 int64_t FRM = MI.getOperand(i: 3).getImm();
26903
26904 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
26905 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
26906
26907 Register FabsReg = MRI.createVirtualRegister(RegClass: RC);
26908 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: FSGNJXOpc), DestReg: FabsReg).addReg(RegNo: SrcReg).addReg(RegNo: SrcReg);
26909
26910 // Compare the FP value to the max value.
26911 Register CmpReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
26912 auto MIB =
26913 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: CmpOpc), DestReg: CmpReg).addReg(RegNo: FabsReg).addReg(RegNo: MaxReg);
26914 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
26915 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
26916
26917 // Insert branch.
26918 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: RISCV::BEQ))
26919 .addReg(RegNo: CmpReg)
26920 .addReg(RegNo: RISCV::X0)
26921 .addMBB(MBB: DoneMBB);
26922
26923 CvtMBB->addSuccessor(Succ: DoneMBB);
26924
26925 // Convert to integer.
26926 Register F2IReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
26927 MIB = BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: F2IOpc), DestReg: F2IReg).addReg(RegNo: SrcReg).addImm(Val: FRM);
26928 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
26929 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
26930
26931 // Convert back to FP.
26932 Register I2FReg = MRI.createVirtualRegister(RegClass: RC);
26933 MIB = BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: I2FOpc), DestReg: I2FReg).addReg(RegNo: F2IReg).addImm(Val: FRM);
26934 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
26935 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
26936
26937 // Restore the sign bit.
26938 Register CvtReg = MRI.createVirtualRegister(RegClass: RC);
26939 BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: FSGNJOpc), DestReg: CvtReg).addReg(RegNo: I2FReg).addReg(RegNo: SrcReg);
26940
26941 // Merge the results.
26942 BuildMI(BB&: *DoneMBB, I: DoneMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: RISCV::PHI), DestReg: DstReg)
26943 .addReg(RegNo: SrcReg)
26944 .addMBB(MBB)
26945 .addReg(RegNo: CvtReg)
26946 .addMBB(MBB: CvtMBB);
26947
26948 MI.eraseFromParent();
26949 return DoneMBB;
26950}
26951
26952MachineBasicBlock *
26953RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
26954 MachineBasicBlock *BB) const {
26955 switch (MI.getOpcode()) {
26956 default:
26957 llvm_unreachable("Unexpected instr type to insert");
26958 case RISCV::ReadCounterWide:
26959 assert(!Subtarget.is64Bit() &&
26960 "ReadCounterWide is only to be used on riscv32");
26961 return emitReadCounterWidePseudo(MI, BB);
26962 case RISCV::Select_GPR_Using_CC_GPR:
26963 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
26964 case RISCV::Select_GPR_Using_CC_SImm5_CV:
26965 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
26966 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
26967 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
26968 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
26969 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
26970 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
26971 case RISCV::Select_FPR16_Using_CC_GPR:
26972 case RISCV::Select_FPR16INX_Using_CC_GPR:
26973 case RISCV::Select_FPR32_Using_CC_GPR:
26974 case RISCV::Select_FPR32INX_Using_CC_GPR:
26975 case RISCV::Select_FPR64_Using_CC_GPR:
26976 case RISCV::Select_FPR64INX_Using_CC_GPR:
26977 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
26978 return emitSelectPseudo(MI, BB, Subtarget);
26979 case RISCV::BuildPairF64Pseudo:
26980 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
26981 case RISCV::SplitF64Pseudo:
26982 return emitSplitF64Pseudo(MI, BB, Subtarget);
26983 case RISCV::PseudoQuietFLE_H:
26984 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_H, EqOpcode: RISCV::FEQ_H, Subtarget);
26985 case RISCV::PseudoQuietFLE_H_INX:
26986 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_H_INX, EqOpcode: RISCV::FEQ_H_INX, Subtarget);
26987 case RISCV::PseudoQuietFLT_H:
26988 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_H, EqOpcode: RISCV::FEQ_H, Subtarget);
26989 case RISCV::PseudoQuietFLT_H_INX:
26990 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_H_INX, EqOpcode: RISCV::FEQ_H_INX, Subtarget);
26991 case RISCV::PseudoQuietFLE_S:
26992 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_S, EqOpcode: RISCV::FEQ_S, Subtarget);
26993 case RISCV::PseudoQuietFLE_S_INX:
26994 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_S_INX, EqOpcode: RISCV::FEQ_S_INX, Subtarget);
26995 case RISCV::PseudoQuietFLT_S:
26996 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_S, EqOpcode: RISCV::FEQ_S, Subtarget);
26997 case RISCV::PseudoQuietFLT_S_INX:
26998 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_S_INX, EqOpcode: RISCV::FEQ_S_INX, Subtarget);
26999 case RISCV::PseudoQuietFLE_D:
27000 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D, EqOpcode: RISCV::FEQ_D, Subtarget);
27001 case RISCV::PseudoQuietFLE_D_INX:
27002 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D_INX, EqOpcode: RISCV::FEQ_D_INX, Subtarget);
27003 case RISCV::PseudoQuietFLE_D_IN32X:
27004 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D_IN32X, EqOpcode: RISCV::FEQ_D_IN32X,
27005 Subtarget);
27006 case RISCV::PseudoQuietFLT_D:
27007 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D, EqOpcode: RISCV::FEQ_D, Subtarget);
27008 case RISCV::PseudoQuietFLT_D_INX:
27009 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D_INX, EqOpcode: RISCV::FEQ_D_INX, Subtarget);
27010 case RISCV::PseudoQuietFLT_D_IN32X:
27011 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D_IN32X, EqOpcode: RISCV::FEQ_D_IN32X,
27012 Subtarget);
27013
27014 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
27015 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M1_MASK);
27016 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
27017 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M2_MASK);
27018 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
27019 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M4_MASK);
27020 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
27021 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M8_MASK);
27022 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
27023 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_MF2_MASK);
27024 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
27025 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_MF4_MASK);
27026 case RISCV::PseudoFROUND_H:
27027 case RISCV::PseudoFROUND_H_INX:
27028 case RISCV::PseudoFROUND_S:
27029 case RISCV::PseudoFROUND_S_INX:
27030 case RISCV::PseudoFROUND_D:
27031 case RISCV::PseudoFROUND_D_INX:
27032 case RISCV::PseudoFROUND_D_IN32X:
27033 return emitFROUND(MI, MBB: BB, Subtarget);
27034 case RISCV::PROBED_STACKALLOC_DYN:
27035 return emitDynamicProbedAlloc(MI, MBB: BB);
27036 case TargetOpcode::STATEPOINT:
27037 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
27038 // while jal call instruction (where statepoint will be lowered at the end)
27039 // has implicit def. This def is early-clobber as it will be set at
27040 // the moment of the call and earlier than any use is read.
27041 // Add this implicit dead def here as a workaround.
27042 MI.addOperand(MF&: *MI.getMF(),
27043 Op: MachineOperand::CreateReg(
27044 Reg: RISCV::X1, /*isDef*/ true,
27045 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
27046 /*isUndef*/ false, /*isEarlyClobber*/ true));
27047 [[fallthrough]];
27048 case TargetOpcode::STACKMAP:
27049 case TargetOpcode::PATCHPOINT:
27050 if (!Subtarget.is64Bit())
27051 reportFatalUsageError(reason: "STACKMAP, PATCHPOINT and STATEPOINT are only "
27052 "supported on 64-bit targets");
27053 return emitPatchPoint(MI, MBB: BB);
27054 }
27055}
27056
27057void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
27058 SDNode *Node) const {
27059 // If instruction defines FRM operand, conservatively set it as non-dead to
27060 // express data dependency with FRM users and prevent incorrect instruction
27061 // reordering.
27062 if (auto *FRMDef = MI.findRegisterDefOperand(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
27063 FRMDef->setIsDead(false);
27064 return;
27065 }
27066 // Add FRM dependency to any instructions with dynamic rounding mode.
27067 int Idx = RISCV::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: RISCV::OpName::frm);
27068 if (Idx < 0) {
27069 // Vector pseudos have FRM index indicated by TSFlags.
27070 Idx = RISCVII::getFRMOpNum(Desc: MI.getDesc());
27071 if (Idx < 0)
27072 return;
27073 }
27074 if (MI.getOperand(i: Idx).getImm() != RISCVFPRndMode::DYN)
27075 return;
27076 // If the instruction already reads FRM, don't add another read.
27077 if (MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr))
27078 return;
27079 MI.addOperand(
27080 Op: MachineOperand::CreateReg(Reg: RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
27081}
27082
27083// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
27084// values.
27085static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
27086 const CCValAssign &VA, const SDLoc &DL,
27087 const RISCVSubtarget &Subtarget) {
27088 if (VA.needsCustom()) {
27089 if (VA.getLocVT().isInteger() &&
27090 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
27091 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT: VA.getValVT(), Operand: Val);
27092 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
27093 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Val);
27094 if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
27095 return convertFromScalableVector(VT: VA.getValVT(), V: Val, DAG, Subtarget);
27096 llvm_unreachable("Unexpected Custom handling.");
27097 }
27098
27099 switch (VA.getLocInfo()) {
27100 default:
27101 llvm_unreachable("Unexpected CCValAssign::LocInfo");
27102 case CCValAssign::Full:
27103 break;
27104 case CCValAssign::BCvt:
27105 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
27106 break;
27107 }
27108 return Val;
27109}
27110
27111// The caller is responsible for loading the full value if the argument is
27112// passed with CCValAssign::Indirect.
27113static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
27114 const CCValAssign &VA, const SDLoc &DL,
27115 const ISD::InputArg &In,
27116 const RISCVTargetLowering &TLI) {
27117 MachineFunction &MF = DAG.getMachineFunction();
27118 MachineRegisterInfo &RegInfo = MF.getRegInfo();
27119 EVT LocVT = VA.getLocVT();
27120 SDValue Val;
27121 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
27122 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
27123 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
27124 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
27125
27126 // If input is sign extended from 32 bits, note it for the RISCVOptWInstrs
27127 // pass.
27128 if (In.isOrigArg()) {
27129 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
27130 if (OrigArg->getType()->isIntegerTy()) {
27131 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
27132 // An input zero extended from i31 can also be considered sign extended.
27133 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
27134 (BitWidth < 32 && In.Flags.isZExt())) {
27135 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
27136 RVFI->addSExt32Register(Reg: VReg);
27137 }
27138 }
27139 }
27140
27141 if (VA.getLocInfo() == CCValAssign::Indirect)
27142 return Val;
27143
27144 return convertLocVTToValVT(DAG, Val, VA, DL, Subtarget: TLI.getSubtarget());
27145}
27146
27147static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
27148 const CCValAssign &VA, const SDLoc &DL,
27149 const RISCVSubtarget &Subtarget) {
27150 EVT LocVT = VA.getLocVT();
27151
27152 if (VA.needsCustom()) {
27153 if (LocVT.isInteger() &&
27154 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
27155 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: LocVT, Operand: Val);
27156 if (LocVT == MVT::i64 && VA.getValVT() == MVT::f32)
27157 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Val);
27158 if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
27159 return convertToScalableVector(VT: LocVT, V: Val, DAG, Subtarget);
27160 llvm_unreachable("Unexpected Custom handling.");
27161 }
27162
27163 switch (VA.getLocInfo()) {
27164 default:
27165 llvm_unreachable("Unexpected CCValAssign::LocInfo");
27166 case CCValAssign::Full:
27167 break;
27168 case CCValAssign::BCvt:
27169 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
27170 break;
27171 }
27172 return Val;
27173}
27174
27175// The caller is responsible for loading the full value if the argument is
27176// passed with CCValAssign::Indirect.
27177static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
27178 const CCValAssign &VA, const SDLoc &DL,
27179 const RISCVTargetLowering &TLI) {
27180 MachineFunction &MF = DAG.getMachineFunction();
27181 MachineFrameInfo &MFI = MF.getFrameInfo();
27182 EVT LocVT = VA.getLocVT();
27183 EVT PtrVT = MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0));
27184 int FI = MFI.CreateFixedObject(Size: LocVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
27185 /*IsImmutable=*/true);
27186 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
27187 SDValue Val = DAG.getLoad(
27188 VT: LocVT, dl: DL, Chain, Ptr: FIN,
27189 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
27190
27191 if (VA.getLocInfo() == CCValAssign::Indirect)
27192 return Val;
27193
27194 return convertLocVTToValVT(DAG, Val, VA, DL, Subtarget: TLI.getSubtarget());
27195}
27196
27197static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
27198 const CCValAssign &VA,
27199 const CCValAssign &HiVA,
27200 const SDLoc &DL) {
27201 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
27202 "Unexpected VA");
27203 MachineFunction &MF = DAG.getMachineFunction();
27204 MachineFrameInfo &MFI = MF.getFrameInfo();
27205 MachineRegisterInfo &RegInfo = MF.getRegInfo();
27206
27207 assert(VA.isRegLoc() && "Expected register VA assignment");
27208
27209 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
27210 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
27211 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
27212 SDValue Hi;
27213 if (HiVA.isMemLoc()) {
27214 // Second half of f64 is passed on the stack.
27215 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
27216 /*IsImmutable=*/true);
27217 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
27218 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
27219 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
27220 } else {
27221 // Second half of f64 is passed in another GPR.
27222 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
27223 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
27224 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
27225 }
27226
27227 // For big-endian, swap the order of Lo and Hi when building the pair.
27228 const RISCVSubtarget &Subtarget = DAG.getSubtarget<RISCVSubtarget>();
27229 if (!Subtarget.isLittleEndian())
27230 std::swap(a&: Lo, b&: Hi);
27231
27232 return DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
27233}
27234
27235static SDValue unpackGPRVecOnRV32(SelectionDAG &DAG, SDValue Chain,
27236 const CCValAssign &VA,
27237 const CCValAssign &HiVA, const SDLoc &DL) {
27238 MachineFunction &MF = DAG.getMachineFunction();
27239 MachineFrameInfo &MFI = MF.getFrameInfo();
27240 MachineRegisterInfo &RegInfo = MF.getRegInfo();
27241
27242 assert(VA.isRegLoc() && "Expected register VA assignment");
27243
27244 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
27245 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
27246 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
27247 SDValue Hi;
27248 if (HiVA.isMemLoc()) {
27249 // Second half of f64 is passed on the stack.
27250 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
27251 /*IsImmutable=*/true);
27252 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
27253 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
27254 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
27255 } else {
27256 // Second half of f64 is passed in another GPR.
27257 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
27258 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
27259 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
27260 }
27261
27262 return DAG.getNode(Opcode: RISCVISD::BuildPairGPRVec, DL, VT: VA.getValVT(), N1: Lo, N2: Hi);
27263}
27264
27265// Transform physical registers into virtual registers.
27266SDValue RISCVTargetLowering::LowerFormalArguments(
27267 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
27268 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
27269 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
27270
27271 MachineFunction &MF = DAG.getMachineFunction();
27272
27273 switch (CallConv) {
27274 default:
27275 reportFatalUsageError(reason: "Unsupported calling convention");
27276 case CallingConv::C:
27277 case CallingConv::Fast:
27278 case CallingConv::PreserveMost:
27279 case CallingConv::GRAAL:
27280 case CallingConv::RISCV_VectorCall:
27281#define CC_VLS_CASE(ABI_VLEN) case CallingConv::RISCV_VLSCall_##ABI_VLEN:
27282 CC_VLS_CASE(32)
27283 CC_VLS_CASE(64)
27284 CC_VLS_CASE(128)
27285 CC_VLS_CASE(256)
27286 CC_VLS_CASE(512)
27287 CC_VLS_CASE(1024)
27288 CC_VLS_CASE(2048)
27289 CC_VLS_CASE(4096)
27290 CC_VLS_CASE(8192)
27291 CC_VLS_CASE(16384)
27292 CC_VLS_CASE(32768)
27293 CC_VLS_CASE(65536)
27294#undef CC_VLS_CASE
27295 break;
27296 case CallingConv::GHC:
27297 if (Subtarget.hasStdExtE())
27298 reportFatalUsageError(reason: "GHC calling convention is not supported on RVE!");
27299 if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx())
27300 reportFatalUsageError(reason: "GHC calling convention requires the (Zfinx/F) and "
27301 "(Zdinx/D) instruction set extensions");
27302 }
27303
27304 const Function &Func = MF.getFunction();
27305 if (Func.hasFnAttribute(Kind: "interrupt")) {
27306 if (!Func.arg_empty())
27307 reportFatalUsageError(
27308 reason: "Functions with the interrupt attribute cannot have arguments!");
27309
27310 StringRef Kind =
27311 MF.getFunction().getFnAttribute(Kind: "interrupt").getValueAsString();
27312
27313 constexpr StringLiteral SupportedInterruptKinds[] = {
27314 "machine",
27315 "supervisor",
27316 "rnmi",
27317 "qci-nest",
27318 "qci-nonest",
27319 "SiFive-CLIC-preemptible",
27320 "SiFive-CLIC-stack-swap",
27321 "SiFive-CLIC-preemptible-stack-swap",
27322 };
27323 if (!llvm::is_contained(Range: SupportedInterruptKinds, Element: Kind))
27324 reportFatalUsageError(
27325 reason: "Function interrupt attribute argument not supported!");
27326
27327 if (Kind.starts_with(Prefix: "qci-") && !Subtarget.hasVendorXqciint())
27328 reportFatalUsageError(
27329 reason: "'qci-*' interrupt kinds require Xqciint extension");
27330
27331 if (Kind.starts_with(Prefix: "SiFive-CLIC-") && !Subtarget.hasVendorXSfmclic())
27332 reportFatalUsageError(
27333 reason: "'SiFive-CLIC-*' interrupt kinds require XSfmclic extension");
27334
27335 if (Kind == "rnmi" && !Subtarget.hasStdExtSmrnmi())
27336 reportFatalUsageError(reason: "'rnmi' interrupt kind requires Srnmi extension");
27337 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
27338 if (Kind.starts_with(Prefix: "SiFive-CLIC-preemptible") && TFI->hasFP(MF))
27339 Func.getContext().diagnose(DI: DiagnosticInfoUnsupported{
27340 Func,
27341 "'SiFive-CLIC-preemptible' interrupt functions cannot have a frame "
27342 "pointer"});
27343 }
27344
27345 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
27346 MVT XLenVT = Subtarget.getXLenVT();
27347 unsigned XLenInBytes = Subtarget.getXLen() / 8;
27348
27349 // Check if this function has any musttail calls. If so, incoming indirect
27350 // arg pointers must be saved in virtual registers so they survive across
27351 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
27352 // when needed to avoid adding register pressure to non-musttail functions.
27353 bool HasMusttail = llvm::any_of(Range: Func, P: [](const BasicBlock &BB) {
27354 return llvm::any_of(Range: BB, P: [](const Instruction &I) {
27355 if (const auto *CI = dyn_cast<CallInst>(Val: &I))
27356 return CI->isMustTailCall();
27357 return false;
27358 });
27359 });
27360 // Used with vargs to accumulate store chains.
27361 std::vector<SDValue> OutChains;
27362
27363 // Assign locations to all of the incoming arguments.
27364 SmallVector<CCValAssign, 16> ArgLocs;
27365 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
27366
27367 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_RISCV);
27368
27369 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
27370 CCValAssign &VA = ArgLocs[i];
27371 SDValue ArgValue;
27372 // Passing f64 on RV32D with a soft float ABI must be handled as a special
27373 // case.
27374 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
27375 assert(VA.needsCustom());
27376 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
27377 } else if (VA.getLocVT() == MVT::i32 &&
27378 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT()) &&
27379 VA.getLocInfo() != CCValAssign::Indirect) {
27380 assert(VA.needsCustom());
27381 ArgValue = unpackGPRVecOnRV32(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
27382 } else if (VA.isRegLoc())
27383 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
27384 else
27385 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL, TLI: *this);
27386
27387 if (VA.getLocInfo() == CCValAssign::Indirect) {
27388 // If the original argument was split and passed by reference (e.g. i128
27389 // on RV32), we need to load all parts of it here (using the same
27390 // address). Vectors may be partly split to registers and partly to the
27391 // stack, in which case the base address is partly offset and subsequent
27392 // stores are relative to that.
27393 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
27394 PtrInfo: MachinePointerInfo()));
27395 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
27396 if (HasMusttail) {
27397 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
27398 Register VReg =
27399 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
27400 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VReg, N: ArgValue);
27401 RVFI->setIncomingIndirectArg(ArgIndex, Reg: VReg);
27402 }
27403 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
27404 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
27405 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
27406 CCValAssign &PartVA = ArgLocs[i + 1];
27407 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
27408 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
27409 if (PartVA.getValVT().isScalableVector())
27410 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
27411 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
27412 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
27413 PtrInfo: MachinePointerInfo()));
27414 ++i;
27415 ++InsIdx;
27416 }
27417 continue;
27418 }
27419 InVals.push_back(Elt: ArgValue);
27420 }
27421
27422 if (any_of(Range&: ArgLocs,
27423 P: [](CCValAssign &VA) { return VA.getLocVT().isScalableVector(); }))
27424 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
27425
27426 if (IsVarArg) {
27427 ArrayRef<MCPhysReg> ArgRegs = RISCV::getArgGPRs(STI: Subtarget);
27428 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
27429 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
27430 MachineFrameInfo &MFI = MF.getFrameInfo();
27431 MachineRegisterInfo &RegInfo = MF.getRegInfo();
27432 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
27433
27434 // Size of the vararg save area. For now, the varargs save area is either
27435 // zero or large enough to hold a0-a7.
27436 int VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
27437 int FI;
27438
27439 // If all registers are allocated, then all varargs must be passed on the
27440 // stack and we don't need to save any argregs.
27441 if (VarArgsSaveSize == 0) {
27442 int VaArgOffset = CCInfo.getStackSize();
27443 FI = MFI.CreateFixedObject(Size: XLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
27444 } else {
27445 int VaArgOffset = -VarArgsSaveSize;
27446 FI = MFI.CreateFixedObject(Size: VarArgsSaveSize, SPOffset: VaArgOffset, IsImmutable: true);
27447
27448 // If saving an odd number of registers then create an extra stack slot to
27449 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
27450 // offsets to even-numbered registers remain 2*XLEN-aligned.
27451 if (Idx % 2) {
27452 MFI.CreateFixedObject(
27453 Size: XLenInBytes, SPOffset: VaArgOffset - static_cast<int>(XLenInBytes), IsImmutable: true);
27454 VarArgsSaveSize += XLenInBytes;
27455 }
27456
27457 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
27458
27459 // Copy the integer registers that may have been used for passing varargs
27460 // to the vararg save area.
27461 for (unsigned I = Idx; I < ArgRegs.size(); ++I) {
27462 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
27463 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
27464 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: XLenVT);
27465 SDValue Store = DAG.getStore(
27466 Chain, dl: DL, Val: ArgValue, Ptr: FIN,
27467 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: (I - Idx) * XLenInBytes));
27468 OutChains.push_back(x: Store);
27469 FIN =
27470 DAG.getMemBasePlusOffset(Base: FIN, Offset: TypeSize::getFixed(ExactSize: XLenInBytes), DL);
27471 }
27472 }
27473
27474 // Record the frame index of the first variable argument
27475 // which is a value necessary to VASTART.
27476 RVFI->setVarArgsFrameIndex(FI);
27477 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
27478 }
27479
27480 // All stores are grouped in one node to allow the matching between
27481 // the size of Ins and InVals. This only happens for vararg functions.
27482 if (!OutChains.empty()) {
27483 assert(IsVarArg && "Only variadic functions should have OutChains");
27484 OutChains.push_back(x: Chain);
27485 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
27486 }
27487
27488 return Chain;
27489}
27490
27491/// isEligibleForTailCallOptimization - Check whether the call is eligible
27492/// for tail call optimization.
27493/// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
27494bool RISCVTargetLowering::isEligibleForTailCallOptimization(
27495 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
27496 const SmallVector<CCValAssign, 16> &ArgLocs) const {
27497
27498 auto CalleeCC = CLI.CallConv;
27499 auto &Outs = CLI.Outs;
27500 auto &Caller = MF.getFunction();
27501 auto CallerCC = Caller.getCallingConv();
27502
27503 // Exception-handling functions need a special set of instructions to
27504 // indicate a return to the hardware. Tail-calling another function would
27505 // probably break this.
27506 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
27507 // should be expanded as new function attributes are introduced.
27508 if (Caller.hasFnAttribute(Kind: "interrupt"))
27509 return false;
27510
27511 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
27512
27513 // Byval parameters hand the function a pointer directly into the stack area
27514 // we want to reuse during a tail call. Working around this *is* possible
27515 // but less efficient and uglier in LowerCall. For musttail, there is no
27516 // workaround today: a byval arg requires a local copy that becomes invalid
27517 // after the tail call deallocates the caller's frame, so rejecting here
27518 // (and triggering reportFatalInternalError in LowerCall) is safer than
27519 // miscompiling.
27520 for (auto &Arg : Outs)
27521 if (Arg.Flags.isByVal())
27522 return false;
27523
27524 // musttail bypasses the remaining checks: the checks either reject cases
27525 // we handle specially (indirect args are forwarded via incoming pointers,
27526 // stack-passed args reuse the matching incoming layout, sret is forwarded
27527 // like any other pointer arg) or are optimizations not applicable to
27528 // mandatory tail calls.
27529 if (IsMustTail)
27530 return true;
27531
27532 // Do not tail call opt if the stack is used to pass parameters.
27533 if (CCInfo.getStackSize() != 0)
27534 return false;
27535
27536 // Do not tail call opt if any parameters need to be passed indirectly.
27537 // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
27538 // passed indirectly. The caller allocates stack space for the value and
27539 // passes a pointer. On a tail call the caller's frame is deallocated
27540 // before the callee executes, leaving the pointer dangling.
27541 for (auto &VA : ArgLocs)
27542 if (VA.getLocInfo() == CCValAssign::Indirect)
27543 return false;
27544
27545 // Do not tail call opt if either caller or callee uses struct return
27546 // semantics.
27547 auto IsCallerStructRet = Caller.hasStructRetAttr();
27548 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
27549 if (IsCallerStructRet || IsCalleeStructRet)
27550 return false;
27551
27552 // The callee has to preserve all registers the caller needs to preserve.
27553 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
27554 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
27555 if (CalleeCC != CallerCC) {
27556 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
27557 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
27558 return false;
27559 }
27560
27561 return true;
27562}
27563
27564static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
27565 return DAG.getDataLayout().getPrefTypeAlign(
27566 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
27567}
27568
27569// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
27570// and output parameter nodes.
27571SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
27572 SmallVectorImpl<SDValue> &InVals) const {
27573 SelectionDAG &DAG = CLI.DAG;
27574 SDLoc &DL = CLI.DL;
27575 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
27576 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
27577 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
27578 SDValue Chain = CLI.Chain;
27579 SDValue Callee = CLI.Callee;
27580 bool &IsTailCall = CLI.IsTailCall;
27581 CallingConv::ID CallConv = CLI.CallConv;
27582 bool IsVarArg = CLI.IsVarArg;
27583 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
27584 MVT XLenVT = Subtarget.getXLenVT();
27585 const CallBase *CB = CLI.CB;
27586
27587 MachineFunction &MF = DAG.getMachineFunction();
27588 MachineFunction::CallSiteInfo CSInfo;
27589
27590 // Set type id for call site info.
27591 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
27592
27593 // Analyze the operands of the call, assigning locations to each operand.
27594 SmallVector<CCValAssign, 16> ArgLocs;
27595 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
27596
27597 switch (CallConv) {
27598 case CallingConv::GHC:
27599 if (Subtarget.hasStdExtE())
27600 reportFatalUsageError(reason: "GHC calling convention is not supported on RVE!");
27601 break;
27602 }
27603
27604 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_RISCV);
27605
27606 // Check if it's really possible to do a tail call.
27607 if (IsTailCall)
27608 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
27609
27610 if (IsTailCall)
27611 ++NumTailCalls;
27612 else if (CLI.CB && CLI.CB->isMustTailCall())
27613 reportFatalInternalError(reason: "failed to perform tail call elimination on a "
27614 "call site marked musttail");
27615
27616 // Get a count of how many bytes are to be pushed on the stack.
27617 unsigned NumBytes = ArgCCInfo.getStackSize();
27618
27619 // Create local copies for byval args
27620 SmallVector<SDValue, 8> ByValArgs;
27621 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
27622 ISD::ArgFlagsTy Flags = Outs[i].Flags;
27623 if (!Flags.isByVal())
27624 continue;
27625
27626 SDValue Arg = OutVals[i];
27627 unsigned Size = Flags.getByValSize();
27628 Align Alignment = Flags.getNonZeroByValAlign();
27629
27630 int FI =
27631 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
27632 SDValue FIPtr = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
27633 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: XLenVT);
27634
27635 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FIPtr, Src: Arg, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
27636 /*IsVolatile=*/isVol: false,
27637 /*AlwaysInline=*/false, /*CI*/ nullptr, OverrideTailCall: IsTailCall,
27638 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
27639 ByValArgs.push_back(Elt: FIPtr);
27640 }
27641
27642 if (!IsTailCall)
27643 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
27644
27645 // Copy argument values to their designated locations.
27646 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
27647 SmallVector<SDValue, 8> MemOpChains;
27648 SDValue StackPtr;
27649 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
27650 ++i, ++OutIdx) {
27651 CCValAssign &VA = ArgLocs[i];
27652 SDValue ArgValue = OutVals[OutIdx];
27653 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
27654
27655 // Handle passing f64 on RV32D with a soft float ABI as a special case.
27656 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
27657 assert(VA.isRegLoc() && "Expected register VA assignment");
27658 assert(VA.needsCustom());
27659 SDValue SplitF64 = DAG.getNode(
27660 Opcode: RISCVISD::SplitF64, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
27661 SDValue Lo = SplitF64.getValue(R: 0);
27662 SDValue Hi = SplitF64.getValue(R: 1);
27663
27664 // For big-endian, swap the order of Lo and Hi when passing.
27665 if (!Subtarget.isLittleEndian())
27666 std::swap(a&: Lo, b&: Hi);
27667
27668 Register RegLo = VA.getLocReg();
27669 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
27670
27671 // Get the CCValAssign for the Hi part.
27672 CCValAssign &HiVA = ArgLocs[++i];
27673
27674 if (HiVA.isMemLoc()) {
27675 // Second half of f64 is passed on the stack.
27676 if (!StackPtr.getNode())
27677 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
27678 SDValue Address =
27679 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
27680 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
27681 // Emit the store.
27682 MemOpChains.push_back(Elt: DAG.getStore(
27683 Chain, dl: DL, Val: Hi, Ptr: Address,
27684 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
27685 } else {
27686 // Second half of f64 is passed in another GPR.
27687 Register RegHigh = HiVA.getLocReg();
27688 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
27689 }
27690 continue;
27691 }
27692
27693 // Handle passing 64-bit vector on RV32 as a special case.
27694 if (VA.getLocVT() == MVT::i32 &&
27695 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT()) &&
27696 VA.getLocInfo() != CCValAssign::Indirect) {
27697 assert(VA.isRegLoc() && "Expected register VA assignment");
27698 assert(VA.needsCustom());
27699 SDValue SplitGPRVec =
27700 DAG.getNode(Opcode: RISCVISD::SplitGPRVec, DL,
27701 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
27702 SDValue Lo = SplitGPRVec.getValue(R: 0);
27703 SDValue Hi = SplitGPRVec.getValue(R: 1);
27704
27705 Register RegLo = VA.getLocReg();
27706 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
27707
27708 // Get the CCValAssign for the Hi part.
27709 CCValAssign &HiVA = ArgLocs[++i];
27710
27711 if (HiVA.isMemLoc()) {
27712 // Second half of vector is passed on the stack.
27713 if (!StackPtr.getNode())
27714 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
27715 SDValue Address =
27716 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
27717 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
27718 // Emit the store.
27719 MemOpChains.push_back(Elt: DAG.getStore(
27720 Chain, dl: DL, Val: Hi, Ptr: Address,
27721 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
27722 } else {
27723 // Second half of vector is passed in another GPR.
27724 Register RegHigh = HiVA.getLocReg();
27725 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
27726 }
27727 continue;
27728 }
27729
27730 // Promote the value if needed.
27731 // For now, only handle fully promoted and indirect arguments.
27732 if (VA.getLocInfo() == CCValAssign::Indirect) {
27733 // For musttail calls, reuse incoming indirect pointers instead of
27734 // creating new stack temporaries. The incoming pointers point to the
27735 // caller's caller's frame, which remains valid after a tail call.
27736 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
27737 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
27738 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
27739
27740 // Resolve which formal parameter is being passed at this call
27741 // position.
27742 //
27743 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
27744 // but Outs[].OrigArgIndex is an index into a filtered arg list
27745 // (empty types removed, via CallLoweringInfo in the target-
27746 // independent layer). IncomingIndirectArgs is keyed by the
27747 // caller's unfiltered Argument::getArgNo(), so we have to walk
27748 // the caller's formals (same filter) to translate the index.
27749 // This target-independent asymmetry should be normalized so
27750 // backends do not need to re-derive the mapping.
27751 //
27752 // Steps:
27753 // 1. Find the call operand at filtered position CallArgIdx.
27754 // 2. If it is an Argument, use getArgNo() directly (same filter
27755 // for caller formals and call operands).
27756 // 3. Otherwise (computed value), walk the caller's formals and
27757 // skip empty types to map the filtered index to getArgNo().
27758 const Argument *FormalArg = nullptr;
27759 unsigned FilteredIdx = 0;
27760 for (const auto &CallArg : CLI.CB->args()) {
27761 if (CallArg->getType()->isEmptyTy())
27762 continue;
27763 if (FilteredIdx == CallArgIdx) {
27764 FormalArg = dyn_cast<Argument>(Val: CallArg);
27765 break;
27766 }
27767 ++FilteredIdx;
27768 }
27769
27770 // For forwarded args, getArgNo() gives the unfiltered index directly.
27771 // For computed args, walk the caller's formals to resolve it.
27772 unsigned FormalArgIdx = CallArgIdx;
27773 if (FormalArg) {
27774 FormalArgIdx = FormalArg->getArgNo();
27775 } else {
27776 FilteredIdx = 0;
27777 for (const auto &Arg : MF.getFunction().args()) {
27778 if (Arg.getType()->isEmptyTy())
27779 continue;
27780 if (FilteredIdx == CallArgIdx) {
27781 FormalArgIdx = Arg.getArgNo();
27782 break;
27783 }
27784 ++FilteredIdx;
27785 }
27786 }
27787
27788 Register VReg = RVFI->getIncomingIndirectArg(ArgIndex: FormalArgIdx);
27789 SDValue CopyOp = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: PtrVT);
27790 // Thread the CopyFromReg output chain through MemOpChains so the
27791 // TokenFactor below sequences the copy with any stores we emit
27792 // for this argument.
27793 MemOpChains.push_back(Elt: CopyOp.getValue(R: 1));
27794 SDValue IncomingPtr = CopyOp;
27795
27796 if (!FormalArg) {
27797 // Computed value: store into the incoming indirect pointer for the
27798 // same-position formal parameter (musttail guarantees matching
27799 // prototypes, so types match). The pointer survives the tail call
27800 // since it points to the caller's caller's frame.
27801 //
27802 // The data-flow edge through IncomingPtr already prevents the
27803 // store from being scheduled before the CopyFromReg. Threading
27804 // CopyOp.getValue(1) (the copy's output chain) into the store
27805 // makes that ordering explicit on the chain edge as well, which
27806 // is the convention for memory ops chaining off their producers.
27807 MemOpChains.push_back(
27808 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: IncomingPtr,
27809 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
27810 // Store any split parts at their respective offsets. Scalable
27811 // vectors need their part offsets multiplied by VSCALE, matching
27812 // the non-musttail spill path below.
27813 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
27814 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
27815 SDValue PartValue = OutVals[OutIdx + 1];
27816 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
27817 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
27818 EVT PartVT = PartValue.getValueType();
27819 if (PartVT.isScalableVector())
27820 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
27821 SDValue Addr =
27822 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: IncomingPtr, N2: Offset);
27823 MemOpChains.push_back(
27824 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: PartValue, Ptr: Addr,
27825 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
27826 ++i;
27827 ++OutIdx;
27828 }
27829 }
27830 ArgValue = IncomingPtr;
27831
27832 // Skip any remaining split parts (for forwarded args, they are
27833 // covered by the forwarded pointer).
27834 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
27835 ++i;
27836 ++OutIdx;
27837 }
27838 } else {
27839 // Store the argument in a stack slot and pass its address.
27840 Align StackAlign =
27841 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
27842 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
27843 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
27844 // If the original argument was split (e.g. i128), we need
27845 // to store the required parts of it here (and pass just one address).
27846 // Vectors may be partly split to registers and partly to the stack, in
27847 // which case the base address is partly offset and subsequent stores
27848 // are relative to that.
27849 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
27850 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
27851 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
27852 // Calculate the total size to store. We don't have access to what
27853 // we're actually storing other than performing the loop and collecting
27854 // the info.
27855 SmallVector<std::pair<SDValue, SDValue>> Parts;
27856 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
27857 SDValue PartValue = OutVals[OutIdx + 1];
27858 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
27859 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
27860 EVT PartVT = PartValue.getValueType();
27861 if (PartVT.isScalableVector())
27862 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
27863 StoredSize += PartVT.getStoreSize();
27864 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
27865 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
27866 ++i;
27867 ++OutIdx;
27868 }
27869 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
27870 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
27871 MemOpChains.push_back(
27872 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
27873 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
27874 for (const auto &Part : Parts) {
27875 SDValue PartValue = Part.first;
27876 SDValue PartOffset = Part.second;
27877 SDValue Address =
27878 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
27879 MemOpChains.push_back(
27880 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
27881 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
27882 }
27883 ArgValue = SpillSlot;
27884 }
27885 } else {
27886 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL, Subtarget);
27887 }
27888
27889 // Use local copy if it is a byval arg.
27890 if (Flags.isByVal())
27891 ArgValue = ByValArgs[j++];
27892
27893 if (VA.isRegLoc()) {
27894 // Queue up the argument copies and emit them at the end.
27895 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
27896
27897 const TargetOptions &Options = DAG.getTarget().Options;
27898 if (Options.EmitCallSiteInfo)
27899 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: i);
27900 } else {
27901 assert(VA.isMemLoc() && "Argument not register or memory");
27902 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
27903 "Tail call not allowed if stack is used for passing parameters");
27904
27905 // Work out the address of the stack slot.
27906 if (!StackPtr.getNode())
27907 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
27908 SDValue Address =
27909 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
27910 N2: DAG.getIntPtrConstant(Val: VA.getLocMemOffset(), DL));
27911
27912 // Emit the store.
27913 MemOpChains.push_back(
27914 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address,
27915 PtrInfo: MachinePointerInfo::getStack(MF, Offset: VA.getLocMemOffset())));
27916 }
27917 }
27918
27919 // Join the stores, which are independent of one another.
27920 if (!MemOpChains.empty())
27921 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
27922
27923 SDValue Glue;
27924
27925 // Build a sequence of copy-to-reg nodes, chained and glued together.
27926 for (auto &Reg : RegsToPass) {
27927 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
27928 Glue = Chain.getValue(R: 1);
27929 }
27930
27931 // Validate that none of the argument registers have been marked as
27932 // reserved, if so report an error. Do the same for the return address if this
27933 // is not a tailcall.
27934 validateCCReservedRegs(Regs: RegsToPass, MF);
27935 if (!IsTailCall && MF.getSubtarget().isRegisterReservedByUser(R: RISCV::X1))
27936 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
27937 MF.getFunction(),
27938 "Return address register required, but has been reserved."});
27939
27940 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
27941 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
27942 // split it and then direct call can be matched by PseudoCALL.
27943 bool CalleeIsLargeExternalSymbol = false;
27944 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
27945 if (auto *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
27946 Callee = getLargeGlobalAddress(N: S, DL, Ty: PtrVT, DAG);
27947 else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
27948 Callee = getLargeExternalSymbol(N: S, DL, Ty: PtrVT, DAG);
27949 CalleeIsLargeExternalSymbol = true;
27950 }
27951 } else if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
27952 const GlobalValue *GV = S->getGlobal();
27953 Callee = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: RISCVII::MO_CALL);
27954 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
27955 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: RISCVII::MO_CALL);
27956 }
27957
27958 // The first call operand is the chain and the second is the target address.
27959 SmallVector<SDValue, 8> Ops;
27960 Ops.push_back(Elt: Chain);
27961 Ops.push_back(Elt: Callee);
27962
27963 // Add argument registers to the end of the list so that they are
27964 // known live into the call.
27965 for (auto &Reg : RegsToPass)
27966 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
27967
27968 // Add a register mask operand representing the call-preserved registers.
27969 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
27970 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
27971 assert(Mask && "Missing call preserved mask for calling convention");
27972 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
27973
27974 // Glue the call to the argument copies, if any.
27975 if (Glue.getNode())
27976 Ops.push_back(Elt: Glue);
27977
27978 assert((!CLI.CFIType || CLI.CB->isIndirectCall()) &&
27979 "Unexpected CFI type for a direct call");
27980
27981 // Emit the call.
27982 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
27983
27984 // Tail calls need software guarded branch (X7) when cf-protection-branch is
27985 // active: PseudoTAIL expands to JALR X0, X6, 0 which lpad rejects, while
27986 // PseudoTAILX7 expands to JALR X0, X7, 0 which lpad accepts.
27987 // Non-tail calls only need SW_GUARDED in Large code model: in non-Large,
27988 // PseudoCALL uses X1 (JALR X1, X1, 0) which lpad accepts as a return address.
27989 bool NeedSWGuardedTail = false;
27990 bool NeedSWGuardedCall = false;
27991 if (MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch() &&
27992 ((CLI.CB && !CLI.CB->isIndirectCall()) || CalleeIsLargeExternalSymbol)) {
27993 NeedSWGuardedTail = true;
27994 if (getTargetMachine().getCodeModel() == CodeModel::Large)
27995 NeedSWGuardedCall = true;
27996 }
27997
27998 // Use special pseudo for returns_twice calls (e.g., setjmp) when
27999 // cf-protection-branch is enabled, to ensure LPAD is inserted after the call.
28000 bool NeedLpadCall =
28001 CLI.CB && CLI.CB->hasFnAttr(Kind: Attribute::ReturnsTwice) &&
28002 MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
28003
28004 if (IsTailCall) {
28005 MF.getFrameInfo().setHasTailCall();
28006 unsigned CallOpc =
28007 NeedSWGuardedTail ? RISCVISD::SW_GUARDED_TAIL : RISCVISD::TAIL;
28008 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL, VTList: NodeTys, Ops);
28009 if (CLI.CFIType)
28010 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
28011 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
28012 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
28013 return Ret;
28014 }
28015
28016 unsigned CallOpc;
28017 // FIXME: Large Code Model + Zicfilp: SW_GUARDED_CALL takes priority over
28018 // LPAD_CALL for returns_twice calls, breaking LPAD alignment.
28019 if (NeedSWGuardedCall)
28020 CallOpc = RISCVISD::SW_GUARDED_CALL;
28021 else if (NeedLpadCall && CLI.CB->isIndirectCall())
28022 CallOpc = RISCVISD::LPAD_CALL_INDIRECT;
28023 else if (NeedLpadCall)
28024 CallOpc = RISCVISD::LPAD_CALL;
28025 else
28026 CallOpc = RISCVISD::CALL;
28027 Chain = DAG.getNode(Opcode: CallOpc, DL, VTList: NodeTys, Ops);
28028 if (CLI.CFIType)
28029 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
28030
28031 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
28032 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
28033 Glue = Chain.getValue(R: 1);
28034
28035 // Mark the end of the call, which is glued to the call itself.
28036 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
28037 Glue = Chain.getValue(R: 1);
28038
28039 // Assign locations to each value returned by this call.
28040 SmallVector<CCValAssign, 16> RVLocs;
28041 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
28042 RetCCInfo.AnalyzeFormalArguments(Ins, Fn: RetCC_RISCV);
28043
28044 // Copy all of the result registers out of their specified physreg.
28045 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
28046 auto &VA = RVLocs[i];
28047 // Copy the value out
28048 SDValue RetValue =
28049 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
28050 // Glue the RetValue to the end of the call sequence
28051 Chain = RetValue.getValue(R: 1);
28052 Glue = RetValue.getValue(R: 2);
28053
28054 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
28055 assert(VA.needsCustom());
28056 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
28057 VT: MVT::i32, Glue);
28058 Chain = RetValue2.getValue(R: 1);
28059 Glue = RetValue2.getValue(R: 2);
28060
28061 // For big-endian, swap the order when building the pair.
28062 SDValue Lo = RetValue;
28063 SDValue Hi = RetValue2;
28064 if (!Subtarget.isLittleEndian())
28065 std::swap(a&: Lo, b&: Hi);
28066
28067 RetValue = DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
28068 } else if (VA.getLocVT() == MVT::i32 &&
28069 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT())) {
28070 assert(VA.needsCustom());
28071 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
28072 VT: MVT::i32, Glue);
28073 Chain = RetValue2.getValue(R: 1);
28074 Glue = RetValue2.getValue(R: 2);
28075
28076 RetValue = DAG.getNode(Opcode: RISCVISD::BuildPairGPRVec, DL, VT: VA.getValVT(),
28077 N1: RetValue, N2: RetValue2);
28078 } else
28079 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL, Subtarget);
28080
28081 InVals.push_back(Elt: RetValue);
28082 }
28083
28084 return Chain;
28085}
28086
28087bool RISCVTargetLowering::CanLowerReturn(
28088 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
28089 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
28090 const Type *RetTy) const {
28091 SmallVector<CCValAssign, 16> RVLocs;
28092 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
28093 return CCInfo.CheckReturn(Outs, Fn: RetCC_RISCV);
28094}
28095
28096SDValue
28097RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
28098 bool IsVarArg,
28099 const SmallVectorImpl<ISD::OutputArg> &Outs,
28100 const SmallVectorImpl<SDValue> &OutVals,
28101 const SDLoc &DL, SelectionDAG &DAG) const {
28102 MachineFunction &MF = DAG.getMachineFunction();
28103
28104 // Stores the assignment of the return value to a location.
28105 SmallVector<CCValAssign, 16> RVLocs;
28106
28107 // Info about the registers and stack slot.
28108 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
28109 *DAG.getContext());
28110
28111 CCInfo.AnalyzeCallOperands(Outs, Fn: RetCC_RISCV);
28112
28113 if (CallConv == CallingConv::GHC && !RVLocs.empty())
28114 reportFatalUsageError(reason: "GHC functions return void only");
28115
28116 SDValue Glue;
28117 SmallVector<SDValue, 4> RetOps(1, Chain);
28118
28119 // Copy the result values into the output registers.
28120 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
28121 SDValue Val = OutVals[OutIdx];
28122 CCValAssign &VA = RVLocs[i];
28123 assert(VA.isRegLoc() && "Can only return in registers!");
28124
28125 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
28126 // Handle returning f64 on RV32D with a soft float ABI.
28127 assert(VA.isRegLoc() && "Expected return via registers");
28128 assert(VA.needsCustom());
28129 SDValue SplitF64 = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
28130 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
28131 SDValue Lo = SplitF64.getValue(R: 0);
28132 SDValue Hi = SplitF64.getValue(R: 1);
28133
28134 // For big-endian, swap the order of Lo and Hi when returning.
28135 if (!Subtarget.isLittleEndian())
28136 std::swap(a&: Lo, b&: Hi);
28137
28138 Register RegLo = VA.getLocReg();
28139 Register RegHi = RVLocs[++i].getLocReg();
28140
28141 if (Subtarget.isRegisterReservedByUser(i: RegLo) ||
28142 Subtarget.isRegisterReservedByUser(i: RegHi))
28143 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
28144 MF.getFunction(),
28145 "Return value register required, but has been reserved."});
28146
28147 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
28148 Glue = Chain.getValue(R: 1);
28149 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
28150 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
28151 Glue = Chain.getValue(R: 1);
28152 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
28153 } else if (VA.getLocVT() == MVT::i32 &&
28154 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT())) {
28155 // Handle returning 64-bit vector on RV32.
28156 assert(VA.isRegLoc() && "Expected return via registers");
28157 assert(VA.needsCustom());
28158 SDValue SplitGPRVec = DAG.getNode(Opcode: RISCVISD::SplitGPRVec, DL,
28159 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
28160 SDValue Lo = SplitGPRVec.getValue(R: 0);
28161 SDValue Hi = SplitGPRVec.getValue(R: 1);
28162
28163 Register RegLo = VA.getLocReg();
28164 Register RegHi = RVLocs[++i].getLocReg();
28165
28166 if (Subtarget.isRegisterReservedByUser(i: RegLo) ||
28167 Subtarget.isRegisterReservedByUser(i: RegHi))
28168 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
28169 MF.getFunction(),
28170 "Return value register required, but has been reserved."});
28171
28172 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
28173 Glue = Chain.getValue(R: 1);
28174 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
28175 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
28176 Glue = Chain.getValue(R: 1);
28177 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
28178 } else {
28179 // Handle a 'normal' return.
28180 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
28181 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
28182
28183 if (Subtarget.isRegisterReservedByUser(i: VA.getLocReg()))
28184 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
28185 MF.getFunction(),
28186 "Return value register required, but has been reserved."});
28187
28188 // Guarantee that all emitted copies are stuck together.
28189 Glue = Chain.getValue(R: 1);
28190 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
28191 }
28192 }
28193
28194 RetOps[0] = Chain; // Update chain.
28195
28196 // Add the glue node if we have it.
28197 if (Glue.getNode()) {
28198 RetOps.push_back(Elt: Glue);
28199 }
28200
28201 if (any_of(Range&: RVLocs,
28202 P: [](CCValAssign &VA) { return VA.getLocVT().isScalableVector(); }))
28203 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
28204
28205 unsigned RetOpc = RISCVISD::RET_GLUE;
28206 // Interrupt service routines use different return instructions.
28207 const Function &Func = DAG.getMachineFunction().getFunction();
28208 if (Func.hasFnAttribute(Kind: "interrupt")) {
28209 if (!Func.getReturnType()->isVoidTy())
28210 reportFatalUsageError(
28211 reason: "Functions with the interrupt attribute must have void return type!");
28212
28213 MachineFunction &MF = DAG.getMachineFunction();
28214 StringRef Kind =
28215 MF.getFunction().getFnAttribute(Kind: "interrupt").getValueAsString();
28216
28217 if (Kind == "supervisor")
28218 RetOpc = RISCVISD::SRET_GLUE;
28219 else if (Kind == "rnmi") {
28220 assert(Subtarget.hasFeature(RISCV::FeatureStdExtSmrnmi) &&
28221 "Need Smrnmi extension for rnmi");
28222 RetOpc = RISCVISD::MNRET_GLUE;
28223 } else if (Kind == "qci-nest" || Kind == "qci-nonest") {
28224 assert(Subtarget.hasFeature(RISCV::FeatureVendorXqciint) &&
28225 "Need Xqciint for qci-(no)nest");
28226 RetOpc = RISCVISD::QC_C_MILEAVERET_GLUE;
28227 } else
28228 RetOpc = RISCVISD::MRET_GLUE;
28229 }
28230
28231 return DAG.getNode(Opcode: RetOpc, DL, VT: MVT::Other, Ops: RetOps);
28232}
28233
28234void RISCVTargetLowering::validateCCReservedRegs(
28235 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
28236 MachineFunction &MF) const {
28237 const Function &F = MF.getFunction();
28238
28239 if (llvm::any_of(Range: Regs, P: [this](auto Reg) {
28240 return Subtarget.isRegisterReservedByUser(i: Reg.first);
28241 }))
28242 F.getContext().diagnose(DI: DiagnosticInfoUnsupported{
28243 F, "Argument register required, but has been reserved."});
28244}
28245
28246// Check if the result of the node is only used as a return value, as
28247// otherwise we can't perform a tail-call.
28248bool RISCVTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
28249 if (N->getNumValues() != 1)
28250 return false;
28251 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
28252 return false;
28253
28254 SDNode *Copy = *N->user_begin();
28255
28256 if (Copy->getOpcode() == ISD::BITCAST) {
28257 return isUsedByReturnOnly(N: Copy, Chain);
28258 }
28259
28260 // TODO: Handle additional opcodes in order to support tail-calling libcalls
28261 // with soft float ABIs.
28262 if (Copy->getOpcode() != ISD::CopyToReg) {
28263 return false;
28264 }
28265
28266 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
28267 // isn't safe to perform a tail call.
28268 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
28269 return false;
28270
28271 // The copy must be used by a RISCVISD::RET_GLUE, and nothing else.
28272 bool HasRet = false;
28273 for (SDNode *Node : Copy->users()) {
28274 if (Node->getOpcode() != RISCVISD::RET_GLUE)
28275 return false;
28276 HasRet = true;
28277 }
28278 if (!HasRet)
28279 return false;
28280
28281 Chain = Copy->getOperand(Num: 0);
28282 return true;
28283}
28284
28285bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
28286 return CI->isTailCall();
28287}
28288
28289/// getConstraintType - Given a constraint letter, return the type of
28290/// constraint it is for this target.
28291RISCVTargetLowering::ConstraintType
28292RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
28293 if (Constraint.size() == 1) {
28294 switch (Constraint[0]) {
28295 default:
28296 break;
28297 case 'f':
28298 case 'R':
28299 return C_RegisterClass;
28300 case 'I':
28301 case 'J':
28302 case 'K':
28303 return C_Immediate;
28304 case 'A':
28305 return C_Memory;
28306 case 's':
28307 case 'S': // A symbolic address
28308 return C_Other;
28309 }
28310 } else {
28311 if (Constraint == "vr" || Constraint == "vd" || Constraint == "vm")
28312 return C_RegisterClass;
28313 if (Constraint == "cr" || Constraint == "cR" || Constraint == "cf")
28314 return C_RegisterClass;
28315 }
28316 return TargetLowering::getConstraintType(Constraint);
28317}
28318
28319std::pair<unsigned, const TargetRegisterClass *>
28320RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
28321 StringRef Constraint,
28322 MVT VT) const {
28323 // First, see if this is a constraint that directly corresponds to a RISC-V
28324 // register class.
28325 if (Constraint.size() == 1) {
28326 switch (Constraint[0]) {
28327 case 'r':
28328 // TODO: Support fixed vectors up to XLen for P extension?
28329 if (VT.isVector())
28330 break;
28331 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
28332 return std::make_pair(x: 0U, y: &RISCV::GPRF16NoX0RegClass);
28333 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
28334 return std::make_pair(x: 0U, y: &RISCV::GPRF32NoX0RegClass);
28335 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
28336 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
28337 return std::make_pair(x: 0U, y: &RISCV::GPRNoX0RegClass);
28338 case 'f':
28339 if (VT == MVT::f16) {
28340 if (Subtarget.hasStdExtZfhmin())
28341 return std::make_pair(x: 0U, y: &RISCV::FPR16RegClass);
28342 if (Subtarget.hasStdExtZhinxmin())
28343 return std::make_pair(x: 0U, y: &RISCV::GPRF16NoX0RegClass);
28344 } else if (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
28345 return std::make_pair(x: 0U, y: &RISCV::FPR16RegClass);
28346 } else if (VT == MVT::f32) {
28347 if (Subtarget.hasStdExtF())
28348 return std::make_pair(x: 0U, y: &RISCV::FPR32RegClass);
28349 if (Subtarget.hasStdExtZfinx())
28350 return std::make_pair(x: 0U, y: &RISCV::GPRF32NoX0RegClass);
28351 } else if (VT == MVT::f64) {
28352 if (Subtarget.hasStdExtD())
28353 return std::make_pair(x: 0U, y: &RISCV::FPR64RegClass);
28354 if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
28355 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
28356 if (Subtarget.hasStdExtZdinx() && Subtarget.is64Bit())
28357 return std::make_pair(x: 0U, y: &RISCV::GPRNoX0RegClass);
28358 }
28359 break;
28360 case 'R':
28361 if (((VT == MVT::i64 || VT == MVT::f64) && !Subtarget.is64Bit()) ||
28362 (VT == MVT::i128 && Subtarget.is64Bit()))
28363 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
28364 break;
28365 default:
28366 break;
28367 }
28368 } else if (Constraint == "vr") {
28369 // Check VM and fractional LMUL first so that those types will use that
28370 // class instead of VR.
28371 for (const auto *RC :
28372 {&RISCV::ZZZ_VMRegClass, &RISCV::ZZZ_VRMF8RegClass,
28373 &RISCV::ZZZ_VRMF4RegClass, &RISCV::ZZZ_VRMF2RegClass,
28374 &RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
28375 &RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN3M1RegClass,
28376 &RISCV::VRN4M1RegClass, &RISCV::VRN5M1RegClass,
28377 &RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass,
28378 &RISCV::VRN8M1RegClass, &RISCV::VRN2M2RegClass,
28379 &RISCV::VRN3M2RegClass, &RISCV::VRN4M2RegClass,
28380 &RISCV::VRN2M4RegClass}) {
28381 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy))
28382 return std::make_pair(x: 0U, y&: RC);
28383
28384 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
28385 MVT ContainerVT = getContainerForFixedLengthVector(VT);
28386 if (TRI->isTypeLegalForClass(RC: *RC, T: ContainerVT))
28387 return std::make_pair(x: 0U, y&: RC);
28388 }
28389 }
28390 } else if (Constraint == "vd") {
28391 // Check VMNoV0 and fractional LMUL first so that those types will use that
28392 // class instead of VRNoV0.
28393 for (const auto *RC :
28394 {&RISCV::ZZZ_VMNoV0RegClass, &RISCV::ZZZ_VRMF8NoV0RegClass,
28395 &RISCV::ZZZ_VRMF4NoV0RegClass, &RISCV::ZZZ_VRMF2NoV0RegClass,
28396 &RISCV::VRNoV0RegClass, &RISCV::VRM2NoV0RegClass,
28397 &RISCV::VRM4NoV0RegClass, &RISCV::VRM8NoV0RegClass,
28398 &RISCV::VRN2M1NoV0RegClass, &RISCV::VRN3M1NoV0RegClass,
28399 &RISCV::VRN4M1NoV0RegClass, &RISCV::VRN5M1NoV0RegClass,
28400 &RISCV::VRN6M1NoV0RegClass, &RISCV::VRN7M1NoV0RegClass,
28401 &RISCV::VRN8M1NoV0RegClass, &RISCV::VRN2M2NoV0RegClass,
28402 &RISCV::VRN3M2NoV0RegClass, &RISCV::VRN4M2NoV0RegClass,
28403 &RISCV::VRN2M4NoV0RegClass}) {
28404 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy))
28405 return std::make_pair(x: 0U, y&: RC);
28406
28407 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
28408 MVT ContainerVT = getContainerForFixedLengthVector(VT);
28409 if (TRI->isTypeLegalForClass(RC: *RC, T: ContainerVT))
28410 return std::make_pair(x: 0U, y&: RC);
28411 }
28412 }
28413 } else if (Constraint == "vm") {
28414 if (TRI->isTypeLegalForClass(RC: RISCV::VMV0RegClass, T: VT.SimpleTy))
28415 return std::make_pair(x: 0U, y: &RISCV::VMV0RegClass);
28416
28417 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
28418 MVT ContainerVT = getContainerForFixedLengthVector(VT);
28419 // VT here might be coerced to vector with i8 elements, so we need to
28420 // check if this is a M1 register here instead of checking VMV0RegClass.
28421 if (TRI->isTypeLegalForClass(RC: RISCV::VRRegClass, T: ContainerVT))
28422 return std::make_pair(x: 0U, y: &RISCV::VMV0RegClass);
28423 }
28424 } else if (Constraint == "cr") {
28425 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
28426 return std::make_pair(x: 0U, y: &RISCV::GPRF16CRegClass);
28427 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
28428 return std::make_pair(x: 0U, y: &RISCV::GPRF32CRegClass);
28429 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
28430 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
28431 if (!VT.isVector())
28432 return std::make_pair(x: 0U, y: &RISCV::GPRCRegClass);
28433 } else if (Constraint == "cR") {
28434 if (((VT == MVT::i64 || VT == MVT::f64) && !Subtarget.is64Bit()) ||
28435 (VT == MVT::i128 && Subtarget.is64Bit()))
28436 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
28437 } else if (Constraint == "cf") {
28438 if (VT == MVT::f16) {
28439 if (Subtarget.hasStdExtZfhmin())
28440 return std::make_pair(x: 0U, y: &RISCV::FPR16CRegClass);
28441 if (Subtarget.hasStdExtZhinxmin())
28442 return std::make_pair(x: 0U, y: &RISCV::GPRF16CRegClass);
28443 } else if (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
28444 return std::make_pair(x: 0U, y: &RISCV::FPR16CRegClass);
28445 } else if (VT == MVT::f32) {
28446 if (Subtarget.hasStdExtF())
28447 return std::make_pair(x: 0U, y: &RISCV::FPR32CRegClass);
28448 if (Subtarget.hasStdExtZfinx())
28449 return std::make_pair(x: 0U, y: &RISCV::GPRF32CRegClass);
28450 } else if (VT == MVT::f64) {
28451 if (Subtarget.hasStdExtD())
28452 return std::make_pair(x: 0U, y: &RISCV::FPR64CRegClass);
28453 if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
28454 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
28455 if (Subtarget.hasStdExtZdinx() && Subtarget.is64Bit())
28456 return std::make_pair(x: 0U, y: &RISCV::GPRCRegClass);
28457 }
28458 }
28459
28460 // Clang will correctly decode the usage of register name aliases into their
28461 // official names. However, other frontends like `rustc` do not. This allows
28462 // users of these frontends to use the ABI names for registers in LLVM-style
28463 // register constraints.
28464 unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
28465 .Case(S: "{zero}", Value: RISCV::X0)
28466 .Case(S: "{ra}", Value: RISCV::X1)
28467 .Case(S: "{sp}", Value: RISCV::X2)
28468 .Case(S: "{gp}", Value: RISCV::X3)
28469 .Case(S: "{tp}", Value: RISCV::X4)
28470 .Case(S: "{t0}", Value: RISCV::X5)
28471 .Case(S: "{t1}", Value: RISCV::X6)
28472 .Case(S: "{t2}", Value: RISCV::X7)
28473 .Cases(CaseStrings: {"{s0}", "{fp}"}, Value: RISCV::X8)
28474 .Case(S: "{s1}", Value: RISCV::X9)
28475 .Case(S: "{a0}", Value: RISCV::X10)
28476 .Case(S: "{a1}", Value: RISCV::X11)
28477 .Case(S: "{a2}", Value: RISCV::X12)
28478 .Case(S: "{a3}", Value: RISCV::X13)
28479 .Case(S: "{a4}", Value: RISCV::X14)
28480 .Case(S: "{a5}", Value: RISCV::X15)
28481 .Case(S: "{a6}", Value: RISCV::X16)
28482 .Case(S: "{a7}", Value: RISCV::X17)
28483 .Case(S: "{s2}", Value: RISCV::X18)
28484 .Case(S: "{s3}", Value: RISCV::X19)
28485 .Case(S: "{s4}", Value: RISCV::X20)
28486 .Case(S: "{s5}", Value: RISCV::X21)
28487 .Case(S: "{s6}", Value: RISCV::X22)
28488 .Case(S: "{s7}", Value: RISCV::X23)
28489 .Case(S: "{s8}", Value: RISCV::X24)
28490 .Case(S: "{s9}", Value: RISCV::X25)
28491 .Case(S: "{s10}", Value: RISCV::X26)
28492 .Case(S: "{s11}", Value: RISCV::X27)
28493 .Case(S: "{t3}", Value: RISCV::X28)
28494 .Case(S: "{t4}", Value: RISCV::X29)
28495 .Case(S: "{t5}", Value: RISCV::X30)
28496 .Case(S: "{t6}", Value: RISCV::X31)
28497 .Default(Value: RISCV::NoRegister);
28498 if (XRegFromAlias != RISCV::NoRegister)
28499 return std::make_pair(x&: XRegFromAlias, y: &RISCV::GPRRegClass);
28500
28501 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
28502 // TableGen record rather than the AsmName to choose registers for InlineAsm
28503 // constraints, plus we want to match those names to the widest floating point
28504 // register type available, manually select floating point registers here.
28505 //
28506 // The second case is the ABI name of the register, so that frontends can also
28507 // use the ABI names in register constraint lists.
28508 if (Subtarget.hasStdExtF()) {
28509 unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
28510 .Cases(CaseStrings: {"{f0}", "{ft0}"}, Value: RISCV::F0_F)
28511 .Cases(CaseStrings: {"{f1}", "{ft1}"}, Value: RISCV::F1_F)
28512 .Cases(CaseStrings: {"{f2}", "{ft2}"}, Value: RISCV::F2_F)
28513 .Cases(CaseStrings: {"{f3}", "{ft3}"}, Value: RISCV::F3_F)
28514 .Cases(CaseStrings: {"{f4}", "{ft4}"}, Value: RISCV::F4_F)
28515 .Cases(CaseStrings: {"{f5}", "{ft5}"}, Value: RISCV::F5_F)
28516 .Cases(CaseStrings: {"{f6}", "{ft6}"}, Value: RISCV::F6_F)
28517 .Cases(CaseStrings: {"{f7}", "{ft7}"}, Value: RISCV::F7_F)
28518 .Cases(CaseStrings: {"{f8}", "{fs0}"}, Value: RISCV::F8_F)
28519 .Cases(CaseStrings: {"{f9}", "{fs1}"}, Value: RISCV::F9_F)
28520 .Cases(CaseStrings: {"{f10}", "{fa0}"}, Value: RISCV::F10_F)
28521 .Cases(CaseStrings: {"{f11}", "{fa1}"}, Value: RISCV::F11_F)
28522 .Cases(CaseStrings: {"{f12}", "{fa2}"}, Value: RISCV::F12_F)
28523 .Cases(CaseStrings: {"{f13}", "{fa3}"}, Value: RISCV::F13_F)
28524 .Cases(CaseStrings: {"{f14}", "{fa4}"}, Value: RISCV::F14_F)
28525 .Cases(CaseStrings: {"{f15}", "{fa5}"}, Value: RISCV::F15_F)
28526 .Cases(CaseStrings: {"{f16}", "{fa6}"}, Value: RISCV::F16_F)
28527 .Cases(CaseStrings: {"{f17}", "{fa7}"}, Value: RISCV::F17_F)
28528 .Cases(CaseStrings: {"{f18}", "{fs2}"}, Value: RISCV::F18_F)
28529 .Cases(CaseStrings: {"{f19}", "{fs3}"}, Value: RISCV::F19_F)
28530 .Cases(CaseStrings: {"{f20}", "{fs4}"}, Value: RISCV::F20_F)
28531 .Cases(CaseStrings: {"{f21}", "{fs5}"}, Value: RISCV::F21_F)
28532 .Cases(CaseStrings: {"{f22}", "{fs6}"}, Value: RISCV::F22_F)
28533 .Cases(CaseStrings: {"{f23}", "{fs7}"}, Value: RISCV::F23_F)
28534 .Cases(CaseStrings: {"{f24}", "{fs8}"}, Value: RISCV::F24_F)
28535 .Cases(CaseStrings: {"{f25}", "{fs9}"}, Value: RISCV::F25_F)
28536 .Cases(CaseStrings: {"{f26}", "{fs10}"}, Value: RISCV::F26_F)
28537 .Cases(CaseStrings: {"{f27}", "{fs11}"}, Value: RISCV::F27_F)
28538 .Cases(CaseStrings: {"{f28}", "{ft8}"}, Value: RISCV::F28_F)
28539 .Cases(CaseStrings: {"{f29}", "{ft9}"}, Value: RISCV::F29_F)
28540 .Cases(CaseStrings: {"{f30}", "{ft10}"}, Value: RISCV::F30_F)
28541 .Cases(CaseStrings: {"{f31}", "{ft11}"}, Value: RISCV::F31_F)
28542 .Default(Value: RISCV::NoRegister);
28543 if (FReg != RISCV::NoRegister) {
28544 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
28545 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
28546 unsigned RegNo = FReg - RISCV::F0_F;
28547 unsigned DReg = RISCV::F0_D + RegNo;
28548 return std::make_pair(x&: DReg, y: &RISCV::FPR64RegClass);
28549 }
28550 if (VT == MVT::f32 || VT == MVT::Other)
28551 return std::make_pair(x&: FReg, y: &RISCV::FPR32RegClass);
28552 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
28553 unsigned RegNo = FReg - RISCV::F0_F;
28554 unsigned HReg = RISCV::F0_H + RegNo;
28555 return std::make_pair(x&: HReg, y: &RISCV::FPR16RegClass);
28556 }
28557 }
28558 }
28559
28560 if (Subtarget.hasVInstructions()) {
28561 Register VReg = StringSwitch<Register>(Constraint.lower())
28562 .Case(S: "{v0}", Value: RISCV::V0)
28563 .Case(S: "{v1}", Value: RISCV::V1)
28564 .Case(S: "{v2}", Value: RISCV::V2)
28565 .Case(S: "{v3}", Value: RISCV::V3)
28566 .Case(S: "{v4}", Value: RISCV::V4)
28567 .Case(S: "{v5}", Value: RISCV::V5)
28568 .Case(S: "{v6}", Value: RISCV::V6)
28569 .Case(S: "{v7}", Value: RISCV::V7)
28570 .Case(S: "{v8}", Value: RISCV::V8)
28571 .Case(S: "{v9}", Value: RISCV::V9)
28572 .Case(S: "{v10}", Value: RISCV::V10)
28573 .Case(S: "{v11}", Value: RISCV::V11)
28574 .Case(S: "{v12}", Value: RISCV::V12)
28575 .Case(S: "{v13}", Value: RISCV::V13)
28576 .Case(S: "{v14}", Value: RISCV::V14)
28577 .Case(S: "{v15}", Value: RISCV::V15)
28578 .Case(S: "{v16}", Value: RISCV::V16)
28579 .Case(S: "{v17}", Value: RISCV::V17)
28580 .Case(S: "{v18}", Value: RISCV::V18)
28581 .Case(S: "{v19}", Value: RISCV::V19)
28582 .Case(S: "{v20}", Value: RISCV::V20)
28583 .Case(S: "{v21}", Value: RISCV::V21)
28584 .Case(S: "{v22}", Value: RISCV::V22)
28585 .Case(S: "{v23}", Value: RISCV::V23)
28586 .Case(S: "{v24}", Value: RISCV::V24)
28587 .Case(S: "{v25}", Value: RISCV::V25)
28588 .Case(S: "{v26}", Value: RISCV::V26)
28589 .Case(S: "{v27}", Value: RISCV::V27)
28590 .Case(S: "{v28}", Value: RISCV::V28)
28591 .Case(S: "{v29}", Value: RISCV::V29)
28592 .Case(S: "{v30}", Value: RISCV::V30)
28593 .Case(S: "{v31}", Value: RISCV::V31)
28594 .Default(Value: RISCV::NoRegister);
28595 if (VReg != RISCV::NoRegister) {
28596 if (TRI->isTypeLegalForClass(RC: RISCV::ZZZ_VMRegClass, T: VT.SimpleTy))
28597 return std::make_pair(x&: VReg, y: &RISCV::ZZZ_VMRegClass);
28598 if (TRI->isTypeLegalForClass(RC: RISCV::VRRegClass, T: VT.SimpleTy))
28599 return std::make_pair(x&: VReg, y: &RISCV::VRRegClass);
28600 for (const auto *RC :
28601 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
28602 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy)) {
28603 VReg = TRI->getMatchingSuperReg(Reg: VReg, SubIdx: RISCV::sub_vrm1_0, RC);
28604 return std::make_pair(x&: VReg, y&: RC);
28605 }
28606 }
28607 }
28608 }
28609
28610 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
28611}
28612
28613InlineAsm::ConstraintCode
28614RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
28615 // Currently only support length 1 constraints.
28616 if (ConstraintCode.size() == 1) {
28617 switch (ConstraintCode[0]) {
28618 case 'A':
28619 return InlineAsm::ConstraintCode::A;
28620 default:
28621 break;
28622 }
28623 }
28624
28625 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
28626}
28627
28628void RISCVTargetLowering::LowerAsmOperandForConstraint(
28629 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
28630 SelectionDAG &DAG) const {
28631 // Currently only support length 1 constraints.
28632 if (Constraint.size() == 1) {
28633 switch (Constraint[0]) {
28634 case 'I':
28635 // Validate & create a 12-bit signed immediate operand.
28636 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
28637 uint64_t CVal = C->getSExtValue();
28638 if (isInt<12>(x: CVal))
28639 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
28640 VT: Subtarget.getXLenVT()));
28641 }
28642 return;
28643 case 'J':
28644 // Validate & create an integer zero operand.
28645 if (isNullConstant(V: Op))
28646 Ops.push_back(
28647 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getXLenVT()));
28648 return;
28649 case 'K':
28650 // Validate & create a 5-bit unsigned immediate operand.
28651 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
28652 uint64_t CVal = C->getZExtValue();
28653 if (isUInt<5>(x: CVal))
28654 Ops.push_back(
28655 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getXLenVT()));
28656 }
28657 return;
28658 case 'S':
28659 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint: "s", Ops, DAG);
28660 return;
28661 default:
28662 break;
28663 }
28664 }
28665 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
28666}
28667
28668Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
28669 Instruction *Inst,
28670 AtomicOrdering Ord) const {
28671 if (Subtarget.hasStdExtZtso()) {
28672 if (isa<LoadInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
28673 return Builder.CreateFence(Ordering: Ord);
28674 return nullptr;
28675 }
28676
28677 if (isa<LoadInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
28678 return Builder.CreateFence(Ordering: Ord);
28679 if (isa<StoreInst>(Val: Inst) && isReleaseOrStronger(AO: Ord))
28680 return Builder.CreateFence(Ordering: AtomicOrdering::Release);
28681 return nullptr;
28682}
28683
28684Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
28685 Instruction *Inst,
28686 AtomicOrdering Ord) const {
28687 if (Subtarget.hasStdExtZtso()) {
28688 if (isa<StoreInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
28689 return Builder.CreateFence(Ordering: Ord);
28690 return nullptr;
28691 }
28692
28693 if (isa<LoadInst>(Val: Inst) && isAcquireOrStronger(AO: Ord))
28694 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire);
28695 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Val: Inst) &&
28696 Ord == AtomicOrdering::SequentiallyConsistent)
28697 return Builder.CreateFence(Ordering: AtomicOrdering::SequentiallyConsistent);
28698 return nullptr;
28699}
28700
28701TargetLowering::AtomicExpansionKind
28702RISCVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
28703 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
28704 // point operations can't be used in an lr/sc sequence without breaking the
28705 // forward-progress guarantee.
28706 if (AI->isFloatingPointOperation() ||
28707 AI->getOperation() == AtomicRMWInst::UIncWrap ||
28708 AI->getOperation() == AtomicRMWInst::UDecWrap ||
28709 AI->getOperation() == AtomicRMWInst::USubCond ||
28710 AI->getOperation() == AtomicRMWInst::USubSat)
28711 return AtomicExpansionKind::CmpXChg;
28712
28713 // Don't expand forced atomics, we want to have __sync libcalls instead.
28714 if (Subtarget.hasForcedAtomics())
28715 return AtomicExpansionKind::None;
28716
28717 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
28718 if (AI->getOperation() == AtomicRMWInst::Nand) {
28719 if (Subtarget.hasStdExtZacas() &&
28720 (Size >= 32 || Subtarget.hasStdExtZabha()))
28721 return AtomicExpansionKind::CmpXChg;
28722 if (Size < 32)
28723 return AtomicExpansionKind::MaskedIntrinsic;
28724 }
28725
28726 if (Size < 32 && !Subtarget.hasStdExtZabha())
28727 return AtomicExpansionKind::MaskedIntrinsic;
28728
28729 return AtomicExpansionKind::None;
28730}
28731
28732static Intrinsic::ID
28733getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
28734 switch (BinOp) {
28735 default:
28736 llvm_unreachable("Unexpected AtomicRMW BinOp");
28737 case AtomicRMWInst::Xchg:
28738 return Intrinsic::riscv_masked_atomicrmw_xchg;
28739 case AtomicRMWInst::Add:
28740 return Intrinsic::riscv_masked_atomicrmw_add;
28741 case AtomicRMWInst::Sub:
28742 return Intrinsic::riscv_masked_atomicrmw_sub;
28743 case AtomicRMWInst::Nand:
28744 return Intrinsic::riscv_masked_atomicrmw_nand;
28745 case AtomicRMWInst::Max:
28746 return Intrinsic::riscv_masked_atomicrmw_max;
28747 case AtomicRMWInst::Min:
28748 return Intrinsic::riscv_masked_atomicrmw_min;
28749 case AtomicRMWInst::UMax:
28750 return Intrinsic::riscv_masked_atomicrmw_umax;
28751 case AtomicRMWInst::UMin:
28752 return Intrinsic::riscv_masked_atomicrmw_umin;
28753 }
28754}
28755
28756Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
28757 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
28758 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
28759 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
28760 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
28761 // mask, as this produces better code than the LR/SC loop emitted by
28762 // int_riscv_masked_atomicrmw_xchg.
28763 if (AI->getOperation() == AtomicRMWInst::Xchg &&
28764 isa<ConstantInt>(Val: AI->getValOperand())) {
28765 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
28766 if (CVal->isZero())
28767 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
28768 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
28769 Align: AI->getAlign(), Ordering: Ord);
28770 if (CVal->isMinusOne())
28771 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
28772 Align: AI->getAlign(), Ordering: Ord);
28773 }
28774
28775 unsigned XLen = Subtarget.getXLen();
28776 Value *Ordering =
28777 Builder.getIntN(N: XLen, C: static_cast<uint64_t>(AI->getOrdering()));
28778 Type *Tys[] = {Builder.getIntNTy(N: XLen), AlignedAddr->getType()};
28779 Function *LrwOpScwLoop = Intrinsic::getOrInsertDeclaration(
28780 M: AI->getModule(),
28781 id: getIntrinsicForMaskedAtomicRMWBinOp(XLen, BinOp: AI->getOperation()), OverloadTys: Tys);
28782
28783 if (XLen == 64) {
28784 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
28785 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
28786 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
28787 }
28788
28789 Value *Result;
28790
28791 // Must pass the shift amount needed to sign extend the loaded value prior
28792 // to performing a signed comparison for min/max. ShiftAmt is the number of
28793 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
28794 // is the number of bits to left+right shift the value in order to
28795 // sign-extend.
28796 if (AI->getOperation() == AtomicRMWInst::Min ||
28797 AI->getOperation() == AtomicRMWInst::Max) {
28798 const DataLayout &DL = AI->getDataLayout();
28799 unsigned ValWidth =
28800 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
28801 Value *SextShamt =
28802 Builder.CreateSub(LHS: Builder.getIntN(N: XLen, C: XLen - ValWidth), RHS: ShiftAmt);
28803 Result = Builder.CreateCall(Callee: LrwOpScwLoop,
28804 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
28805 } else {
28806 Result =
28807 Builder.CreateCall(Callee: LrwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
28808 }
28809
28810 if (XLen == 64)
28811 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
28812 return Result;
28813}
28814
28815TargetLowering::AtomicExpansionKind
28816RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
28817 const AtomicCmpXchgInst *CI) const {
28818 // Don't expand forced atomics, we want to have __sync libcalls instead.
28819 if (Subtarget.hasForcedAtomics())
28820 return AtomicExpansionKind::None;
28821
28822 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
28823 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
28824 (Size == 8 || Size == 16))
28825 return AtomicExpansionKind::MaskedIntrinsic;
28826 return AtomicExpansionKind::None;
28827}
28828
28829Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
28830 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
28831 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
28832 unsigned XLen = Subtarget.getXLen();
28833 Value *Ordering = Builder.getIntN(N: XLen, C: static_cast<uint64_t>(Ord));
28834 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg;
28835 if (XLen == 64) {
28836 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
28837 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
28838 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
28839 }
28840 Type *Tys[] = {Builder.getIntNTy(N: XLen), AlignedAddr->getType()};
28841 Value *Result = Builder.CreateIntrinsic(
28842 ID: CmpXchgIntrID, OverloadTypes: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
28843 if (XLen == 64)
28844 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
28845 return Result;
28846}
28847
28848bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend,
28849 EVT DataVT) const {
28850 // We have indexed loads for all supported EEW types. Indices are always
28851 // zero extended.
28852 return Extend.getOpcode() == ISD::ZERO_EXTEND &&
28853 isTypeLegal(VT: Extend.getValueType()) &&
28854 isTypeLegal(VT: Extend.getOperand(i: 0).getValueType()) &&
28855 Extend.getOperand(i: 0).getValueType().getVectorElementType() != MVT::i1;
28856}
28857
28858bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
28859 EVT VT) const {
28860 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
28861 return false;
28862
28863 switch (FPVT.getSimpleVT().SimpleTy) {
28864 case MVT::f16:
28865 return Subtarget.hasStdExtZfhmin();
28866 case MVT::f32:
28867 return Subtarget.hasStdExtF();
28868 case MVT::f64:
28869 return Subtarget.hasStdExtD();
28870 default:
28871 return false;
28872 }
28873}
28874
28875unsigned RISCVTargetLowering::getJumpTableEncoding() const {
28876 // If we are using the small code model, we can reduce size of jump table
28877 // entry to 4 bytes.
28878 if (Subtarget.is64Bit() && !isPositionIndependent() &&
28879 getTargetMachine().getCodeModel() == CodeModel::Small) {
28880 return MachineJumpTableInfo::EK_Custom32;
28881 }
28882 return TargetLowering::getJumpTableEncoding();
28883}
28884
28885const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
28886 const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
28887 unsigned uid, MCContext &Ctx) const {
28888 assert(Subtarget.is64Bit() && !isPositionIndependent() &&
28889 getTargetMachine().getCodeModel() == CodeModel::Small);
28890 return MCSymbolRefExpr::create(Symbol: MBB->getSymbol(), Ctx);
28891}
28892
28893bool RISCVTargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
28894 SDValue &Offset,
28895 ISD::MemIndexedMode &AM,
28896 SelectionDAG &DAG) const {
28897 // Target does not support indexed loads.
28898 if (!Subtarget.hasVendorXTHeadMemIdx())
28899 return false;
28900
28901 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
28902 return false;
28903
28904 Base = Op->getOperand(Num: 0);
28905 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1))) {
28906 int64_t RHSC = RHS->getSExtValue();
28907 if (Op->getOpcode() == ISD::SUB)
28908 RHSC = -(uint64_t)RHSC;
28909
28910 // The constants that can be encoded in the THeadMemIdx instructions
28911 // are of the form (sign_extend(imm5) << imm2).
28912 bool isLegalIndexedOffset = false;
28913 for (unsigned i = 0; i < 4; i++)
28914 if (isInt<5>(x: RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
28915 isLegalIndexedOffset = true;
28916 break;
28917 }
28918
28919 if (!isLegalIndexedOffset)
28920 return false;
28921
28922 Offset = Op->getOperand(Num: 1);
28923 return true;
28924 }
28925
28926 return false;
28927}
28928
28929bool RISCVTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
28930 SDValue &Offset,
28931 ISD::MemIndexedMode &AM,
28932 SelectionDAG &DAG) const {
28933 EVT VT;
28934 SDValue Ptr;
28935 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
28936 VT = LD->getMemoryVT();
28937 Ptr = LD->getBasePtr();
28938 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
28939 VT = ST->getMemoryVT();
28940 Ptr = ST->getBasePtr();
28941 } else
28942 return false;
28943
28944 if (!getIndexedAddressParts(Op: Ptr.getNode(), Base, Offset, AM, DAG))
28945 return false;
28946
28947 AM = ISD::PRE_INC;
28948 return true;
28949}
28950
28951bool RISCVTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
28952 SDValue &Base,
28953 SDValue &Offset,
28954 ISD::MemIndexedMode &AM,
28955 SelectionDAG &DAG) const {
28956 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
28957 if (Op->getOpcode() != ISD::ADD)
28958 return false;
28959
28960 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N))
28961 Base = LS->getBasePtr();
28962 else
28963 return false;
28964
28965 if (Base == Op->getOperand(Num: 0))
28966 Offset = Op->getOperand(Num: 1);
28967 else if (Base == Op->getOperand(Num: 1))
28968 Offset = Op->getOperand(Num: 0);
28969 else
28970 return false;
28971
28972 AM = ISD::POST_INC;
28973 return true;
28974 }
28975
28976 EVT VT;
28977 SDValue Ptr;
28978 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
28979 VT = LD->getMemoryVT();
28980 Ptr = LD->getBasePtr();
28981 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
28982 VT = ST->getMemoryVT();
28983 Ptr = ST->getBasePtr();
28984 } else
28985 return false;
28986
28987 if (!getIndexedAddressParts(Op, Base, Offset, AM, DAG))
28988 return false;
28989 // Post-indexing updates the base, so it's not a valid transform
28990 // if that's not the same as the load's pointer.
28991 if (Ptr != Base)
28992 return false;
28993
28994 AM = ISD::POST_INC;
28995 return true;
28996}
28997
28998bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
28999 EVT VT) const {
29000 EVT SVT = VT.getScalarType();
29001
29002 if (!SVT.isSimple())
29003 return false;
29004
29005 switch (SVT.getSimpleVT().SimpleTy) {
29006 case MVT::f16:
29007 return VT.isVector() ? Subtarget.hasVInstructionsF16()
29008 : Subtarget.hasStdExtZfhOrZhinx();
29009 case MVT::f32:
29010 return Subtarget.hasStdExtFOrZfinx();
29011 case MVT::f64:
29012 return Subtarget.hasStdExtDOrZdinx();
29013 default:
29014 break;
29015 }
29016
29017 return false;
29018}
29019
29020ISD::NodeType RISCVTargetLowering::getExtendForAtomicCmpSwapArg() const {
29021 // Zacas will use amocas.w which does not require extension.
29022 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
29023}
29024
29025ISD::NodeType RISCVTargetLowering::getExtendForAtomicRMWArg(unsigned Op) const {
29026 // Zaamo will use amo<op>.w which does not require extension.
29027 if (Subtarget.hasStdExtZaamo() || Subtarget.hasForcedAtomics())
29028 return ISD::ANY_EXTEND;
29029
29030 // Zalrsc pseudo expansions with comparison require sign-extension.
29031 assert(Subtarget.hasStdExtZalrsc());
29032 switch (Op) {
29033 case ISD::ATOMIC_LOAD_MIN:
29034 case ISD::ATOMIC_LOAD_MAX:
29035 case ISD::ATOMIC_LOAD_UMIN:
29036 case ISD::ATOMIC_LOAD_UMAX:
29037 return ISD::SIGN_EXTEND;
29038 default:
29039 break;
29040 }
29041 return ISD::ANY_EXTEND;
29042}
29043
29044Register RISCVTargetLowering::getExceptionPointerRegister(
29045 ExceptionHandling EH, const Constant *PersonalityFn) const {
29046 return RISCV::X10;
29047}
29048
29049Register RISCVTargetLowering::getExceptionSelectorRegister(
29050 ExceptionHandling EH, const Constant *PersonalityFn) const {
29051 return RISCV::X11;
29052}
29053
29054bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
29055 // Return false to suppress the unnecessary extensions if the LibCall
29056 // arguments or return value is a float narrower than XLEN on a soft FP ABI.
29057 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
29058 Type.getSizeInBits() < Subtarget.getXLen()))
29059 return false;
29060
29061 return true;
29062}
29063
29064bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(Type *Ty,
29065 bool IsSigned) const {
29066 if (Subtarget.is64Bit() && Ty->isIntegerTy(BitWidth: 32))
29067 return true;
29068
29069 return IsSigned;
29070}
29071
29072bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
29073 SDValue C) const {
29074 // Check integral scalar types.
29075 if (!VT.isScalarInteger())
29076 return false;
29077
29078 // Omit the optimization if the sub target has the M extension and the data
29079 // size exceeds XLen.
29080 const bool HasZmmul = Subtarget.hasStdExtZmmul();
29081 if (HasZmmul && VT.getSizeInBits() > Subtarget.getXLen())
29082 return false;
29083
29084 auto *ConstNode = cast<ConstantSDNode>(Val&: C);
29085 const APInt &Imm = ConstNode->getAPIntValue();
29086
29087 // Don't do this if the Xqciac extension is enabled and the Imm in simm12.
29088 if (Subtarget.hasVendorXqciac() && Imm.isSignedIntN(N: 12))
29089 return false;
29090
29091 // Break the MUL to a SLLI and an ADD/SUB.
29092 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
29093 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
29094 return true;
29095
29096 // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
29097 if (Subtarget.hasShlAdd(ShAmt: 3) && !Imm.isSignedIntN(N: 12) &&
29098 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
29099 (Imm - 8).isPowerOf2()))
29100 return true;
29101
29102 // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
29103 // a pair of LUI/ADDI.
29104 if (!Imm.isSignedIntN(N: 12) && Imm.countr_zero() < 12 &&
29105 ConstNode->hasOneUse()) {
29106 APInt ImmS = Imm.ashr(ShiftAmt: Imm.countr_zero());
29107 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
29108 (1 - ImmS).isPowerOf2())
29109 return true;
29110 }
29111
29112 return false;
29113}
29114
29115bool RISCVTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
29116 SDValue ConstNode) const {
29117 // Let the DAGCombiner decide for vectors.
29118 EVT VT = AddNode.getValueType();
29119 if (VT.isVector())
29120 return true;
29121
29122 // Let the DAGCombiner decide for larger types.
29123 if (VT.getScalarSizeInBits() > Subtarget.getXLen())
29124 return true;
29125
29126 // It is worse if c1 is simm12 while c1*c2 is not.
29127 ConstantSDNode *C1Node = cast<ConstantSDNode>(Val: AddNode.getOperand(i: 1));
29128 ConstantSDNode *C2Node = cast<ConstantSDNode>(Val&: ConstNode);
29129 const APInt &C1 = C1Node->getAPIntValue();
29130 const APInt &C2 = C2Node->getAPIntValue();
29131 if (C1.isSignedIntN(N: 12) && !(C1 * C2).isSignedIntN(N: 12))
29132 return false;
29133
29134 // Default to true and let the DAGCombiner decide.
29135 return true;
29136}
29137
29138bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
29139 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
29140 unsigned *Fast) const {
29141 if (!VT.isVector() || Subtarget.hasStdExtP()) {
29142 if (Fast)
29143 *Fast = Subtarget.enableUnalignedScalarMem();
29144 return Subtarget.enableUnalignedScalarMem();
29145 }
29146
29147 // All vector implementations must support element alignment
29148 EVT ElemVT = VT.getVectorElementType();
29149 if (Alignment >= ElemVT.getStoreSize()) {
29150 if (Fast)
29151 *Fast = 1;
29152 return true;
29153 }
29154
29155 // Note: We lower an unmasked unaligned vector access to an equally sized
29156 // e8 element type access. Given this, we effectively support all unmasked
29157 // misaligned accesses. TODO: Work through the codegen implications of
29158 // allowing such accesses to be formed, and considered fast.
29159 if (Fast)
29160 *Fast = Subtarget.enableUnalignedVectorMem();
29161 return Subtarget.enableUnalignedVectorMem();
29162}
29163
29164EVT RISCVTargetLowering::getOptimalMemOpType(
29165 LLVMContext &Context, const MemOp &Op,
29166 const AttributeList &FuncAttributes) const {
29167 if (!Subtarget.hasVInstructions())
29168 return MVT::Other;
29169
29170 if (FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat))
29171 return MVT::Other;
29172
29173 // We use LMUL1 memory operations here for a non-obvious reason. Our caller
29174 // has an expansion threshold, and we want the number of hardware memory
29175 // operations to correspond roughly to that threshold. LMUL>1 operations
29176 // are typically expanded linearly internally, and thus correspond to more
29177 // than one actual memory operation. Note that store merging and load
29178 // combining will typically form larger LMUL operations from the LMUL1
29179 // operations emitted here, and that's okay because combining isn't
29180 // introducing new memory operations; it's just merging existing ones.
29181 // NOTE: We limit to 1024 bytes to avoid creating an invalid MVT.
29182 const unsigned MinVLenInBytes =
29183 std::min(a: Subtarget.getRealMinVLen() / 8, b: 1024U);
29184
29185 if (Op.size() < MinVLenInBytes)
29186 // TODO: Figure out short memops. For the moment, do the default thing
29187 // which ends up using scalar sequences.
29188 return MVT::Other;
29189
29190 // If the minimum VLEN is less than RISCV::RVVBitsPerBlock we don't support
29191 // fixed vectors.
29192 if (MinVLenInBytes <= RISCV::RVVBytesPerBlock)
29193 return MVT::Other;
29194
29195 // Prefer i8 for non-zero memset as it allows us to avoid materializing
29196 // a large scalar constant and instead use vmv.v.x/i to do the
29197 // broadcast. For everything else, prefer ELenVT to minimize VL and thus
29198 // maximize the chance we can encode the size in the vsetvli.
29199 MVT ELenVT = MVT::getIntegerVT(BitWidth: Subtarget.getELen());
29200 MVT PreferredVT = (Op.isMemset() && !Op.isZeroMemset()) ? MVT::i8 : ELenVT;
29201
29202 // Do we have sufficient alignment for our preferred VT? If not, revert
29203 // to largest size allowed by our alignment criteria.
29204 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
29205 Align RequiredAlign(PreferredVT.getStoreSize());
29206 if (Op.isFixedDstAlign())
29207 RequiredAlign = std::min(a: RequiredAlign, b: Op.getDstAlign());
29208 if (Op.isMemcpyOrMemmove())
29209 RequiredAlign = std::min(a: RequiredAlign, b: Op.getSrcAlign());
29210 PreferredVT = MVT::getIntegerVT(BitWidth: RequiredAlign.value() * 8);
29211 }
29212 return MVT::getVectorVT(VT: PreferredVT, NumElements: MinVLenInBytes/PreferredVT.getStoreSize());
29213}
29214
29215bool RISCVTargetLowering::splitValueIntoRegisterParts(
29216 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
29217 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
29218 bool IsABIRegCopy = CC.has_value();
29219 EVT ValueVT = Val.getValueType();
29220
29221 MVT PairVT = Subtarget.is64Bit() ? MVT::i128 : MVT::i64;
29222 if ((ValueVT == PairVT ||
29223 (!Subtarget.is64Bit() && Subtarget.hasStdExtZdinx() &&
29224 ValueVT == MVT::f64)) &&
29225 NumParts == 1 && PartVT == MVT::Untyped) {
29226 // Pairs in Inline Assembly, f64 in Inline assembly on rv32_zdinx
29227 MVT XLenVT = Subtarget.getXLenVT();
29228 if (ValueVT == MVT::f64)
29229 Val = DAG.getBitcast(VT: MVT::i64, V: Val);
29230 auto [Lo, Hi] = DAG.SplitScalar(N: Val, DL, LoVT: XLenVT, HiVT: XLenVT);
29231 // Always creating an MVT::Untyped part, so always use
29232 // RISCVISD::BuildGPRPair.
29233 Parts[0] = DAG.getNode(Opcode: RISCVISD::BuildGPRPair, DL, VT: PartVT, N1: Lo, N2: Hi);
29234 return true;
29235 }
29236
29237 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
29238 PartVT == MVT::f32) {
29239 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
29240 // nan, and cast to f32.
29241 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
29242 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
29243 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
29244 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
29245 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
29246 Parts[0] = Val;
29247 return true;
29248 }
29249
29250 if (ValueVT.isRISCVVectorTuple() && PartVT.isRISCVVectorTuple()) {
29251#ifndef NDEBUG
29252 unsigned ValNF = ValueVT.getRISCVVectorTupleNumFields();
29253 [[maybe_unused]] unsigned ValLMUL =
29254 divideCeil(ValueVT.getSizeInBits().getKnownMinValue(),
29255 ValNF * RISCV::RVVBitsPerBlock);
29256 unsigned PartNF = PartVT.getRISCVVectorTupleNumFields();
29257 [[maybe_unused]] unsigned PartLMUL =
29258 divideCeil(PartVT.getSizeInBits().getKnownMinValue(),
29259 PartNF * RISCV::RVVBitsPerBlock);
29260 assert(ValNF == PartNF && ValLMUL == PartLMUL &&
29261 "RISC-V vector tuple type only accepts same register class type "
29262 "TUPLE_INSERT");
29263#endif
29264
29265 Val = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
29266 N2: Val, N3: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
29267 Parts[0] = Val;
29268 return true;
29269 }
29270
29271 if (ValueVT.isFixedLengthVector() && PartVT.isScalableVector()) {
29272 ValueVT = getContainerForFixedLengthVector(VT: ValueVT.getSimpleVT());
29273 Val = convertToScalableVector(VT: ValueVT, V: Val, DAG, Subtarget);
29274
29275 LLVMContext &Context = *DAG.getContext();
29276 EVT ValueEltVT = ValueVT.getVectorElementType();
29277 EVT PartEltVT = PartVT.getVectorElementType();
29278 unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinValue();
29279 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
29280 if (PartVTBitSize % ValueVTBitSize == 0) {
29281 assert(PartVTBitSize >= ValueVTBitSize);
29282 // If the element types are different, bitcast to the same element type of
29283 // PartVT first.
29284 // Give an example here, we want copy a <vscale x 1 x i8> value to
29285 // <vscale x 4 x i16>.
29286 // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
29287 // subvector, then we can bitcast to <vscale x 4 x i16>.
29288 if (ValueEltVT != PartEltVT) {
29289 if (PartVTBitSize > ValueVTBitSize) {
29290 unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
29291 assert(Count != 0 && "The number of element should not be zero.");
29292 EVT SameEltTypeVT =
29293 EVT::getVectorVT(Context, VT: ValueEltVT, NumElements: Count, /*IsScalable=*/true);
29294 Val = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SameEltTypeVT), SubVec: Val, Idx: 0);
29295 }
29296 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
29297 } else {
29298 Val = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: PartVT), SubVec: Val, Idx: 0);
29299 }
29300 Parts[0] = Val;
29301 return true;
29302 }
29303 }
29304
29305 return false;
29306}
29307
29308SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
29309 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
29310 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
29311 bool IsABIRegCopy = CC.has_value();
29312
29313 MVT PairVT = Subtarget.is64Bit() ? MVT::i128 : MVT::i64;
29314 if ((ValueVT == PairVT ||
29315 (!Subtarget.is64Bit() && Subtarget.hasStdExtZdinx() &&
29316 ValueVT == MVT::f64)) &&
29317 NumParts == 1 && PartVT == MVT::Untyped) {
29318 // Pairs in Inline Assembly, f64 in Inline assembly on rv32_zdinx
29319 MVT XLenVT = Subtarget.getXLenVT();
29320
29321 SDValue Val = Parts[0];
29322 // Always starting with an MVT::Untyped part, so always use
29323 // RISCVISD::SplitGPRPair
29324 Val = DAG.getNode(Opcode: RISCVISD::SplitGPRPair, DL, VTList: DAG.getVTList(VT1: XLenVT, VT2: XLenVT),
29325 N: Val);
29326 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: PairVT, N1: Val.getValue(R: 0),
29327 N2: Val.getValue(R: 1));
29328 if (ValueVT == MVT::f64)
29329 Val = DAG.getBitcast(VT: ValueVT, V: Val);
29330 return Val;
29331 }
29332
29333 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
29334 PartVT == MVT::f32) {
29335 SDValue Val = Parts[0];
29336
29337 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
29338 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
29339 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
29340 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
29341 return Val;
29342 }
29343
29344 if (ValueVT.isFixedLengthVector() && PartVT.isScalableVector()) {
29345 LLVMContext &Context = *DAG.getContext();
29346 SDValue Val = Parts[0];
29347 EVT ValueEltVT = ValueVT.getVectorElementType();
29348 EVT PartEltVT = PartVT.getVectorElementType();
29349
29350 unsigned ValueVTBitSize =
29351 getContainerForFixedLengthVector(VT: ValueVT.getSimpleVT())
29352 .getSizeInBits()
29353 .getKnownMinValue();
29354
29355 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
29356 if (PartVTBitSize % ValueVTBitSize == 0) {
29357 assert(PartVTBitSize >= ValueVTBitSize);
29358 EVT SameEltTypeVT = ValueVT;
29359 // If the element types are different, convert it to the same element type
29360 // of PartVT.
29361 // Give an example here, we want copy a <vscale x 1 x i8> value from
29362 // <vscale x 4 x i16>.
29363 // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
29364 // then we can extract <vscale x 1 x i8>.
29365 if (ValueEltVT != PartEltVT) {
29366 unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
29367 assert(Count != 0 && "The number of element should not be zero.");
29368 SameEltTypeVT =
29369 EVT::getVectorVT(Context, VT: ValueEltVT, NumElements: Count, /*IsScalable=*/true);
29370 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: SameEltTypeVT, Operand: Val);
29371 }
29372 if (ValueVT.isFixedLengthVector())
29373 Val = convertFromScalableVector(VT: ValueVT, V: Val, DAG, Subtarget);
29374 else
29375 Val = DAG.getExtractSubvector(DL, VT: ValueVT, Vec: Val, Idx: 0);
29376 return Val;
29377 }
29378 }
29379 return SDValue();
29380}
29381
29382bool RISCVTargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
29383 // When aggressively optimizing for code size, we prefer to use a div
29384 // instruction, as it is usually smaller than the alternative sequence.
29385 // TODO: Add vector division?
29386 bool OptSize = Attr.hasFnAttr(Kind: Attribute::MinSize);
29387 return OptSize && !VT.isVector() &&
29388 VT.getSizeInBits() <= getMaxDivRemBitWidthSupported();
29389}
29390
29391void RISCVTargetLowering::finalizeLowering(MachineFunction &MF) const {
29392 MF.getFrameInfo().computeMaxCallFrameSize(MF);
29393 TargetLoweringBase::finalizeLowering(MF);
29394}
29395
29396bool RISCVTargetLowering::preferScalarizeSplat(SDNode *N) const {
29397 // Scalarize zero_ext and sign_ext might stop match to widening instruction in
29398 // some situation.
29399 unsigned Opc = N->getOpcode();
29400 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND)
29401 return false;
29402 return true;
29403}
29404
29405static Value *useTpOffset(IRBuilderBase &IRB, unsigned Offset) {
29406 Module *M = IRB.GetInsertBlock()->getModule();
29407 Function *ThreadPointerFunc = Intrinsic::getOrInsertDeclaration(
29408 M, id: Intrinsic::thread_pointer, OverloadTys: IRB.getPtrTy());
29409 return IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(),
29410 Ptr: IRB.CreateCall(Callee: ThreadPointerFunc), Idx0: Offset);
29411}
29412
29413Value *RISCVTargetLowering::getIRStackGuard(
29414 IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const {
29415 // Fuchsia provides a fixed TLS slot for the stack cookie.
29416 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
29417 if (Subtarget.isTargetFuchsia())
29418 return useTpOffset(IRB, Offset: -0x10);
29419
29420 // Android provides a fixed TLS slot for the stack cookie. See the definition
29421 // of TLS_SLOT_STACK_GUARD in
29422 // https://android.googlesource.com/platform/bionic/+/main/libc/platform/bionic/tls_defines.h
29423 if (Subtarget.isTargetAndroid())
29424 return useTpOffset(IRB, Offset: -0x18);
29425
29426 Module *M = IRB.GetInsertBlock()->getModule();
29427
29428 if (M->getStackProtectorGuard() == "tls") {
29429 // Users must specify the offset explicitly
29430 int Offset = M->getStackProtectorGuardOffset();
29431 return useTpOffset(IRB, Offset);
29432 }
29433
29434 return TargetLowering::getIRStackGuard(IRB, Libcalls);
29435}
29436
29437bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
29438 Align Alignment) const {
29439 if (!Subtarget.hasVInstructions())
29440 return false;
29441
29442 // Only support fixed vectors if we know the minimum vector size.
29443 if (DataType.isFixedLengthVector() && !Subtarget.useRVVForFixedLengthVectors())
29444 return false;
29445
29446 EVT ScalarType = DataType.getScalarType();
29447 if (!isLegalElementTypeForRVV(ScalarTy: ScalarType))
29448 return false;
29449
29450 if (!Subtarget.enableUnalignedVectorMem() &&
29451 Alignment < ScalarType.getStoreSize())
29452 return false;
29453
29454 return true;
29455}
29456
29457bool RISCVTargetLowering::isLegalFirstFaultLoad(EVT DataType,
29458 Align Alignment) const {
29459 if (!Subtarget.hasVInstructions())
29460 return false;
29461
29462 EVT ScalarType = DataType.getScalarType();
29463 if (!isLegalElementTypeForRVV(ScalarTy: ScalarType))
29464 return false;
29465
29466 if (!Subtarget.enableUnalignedVectorMem() &&
29467 Alignment < ScalarType.getStoreSize())
29468 return false;
29469
29470 return true;
29471}
29472
29473MachineInstr *
29474RISCVTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
29475 MachineBasicBlock::instr_iterator &MBBI,
29476 const TargetInstrInfo *TII) const {
29477 assert(MBBI->isCall() && MBBI->getCFIType() &&
29478 "Invalid call instruction for a KCFI check");
29479 assert(is_contained({RISCV::PseudoCALLIndirect, RISCV::PseudoTAILIndirect},
29480 MBBI->getOpcode()));
29481
29482 MachineOperand &Target = MBBI->getOperand(i: 0);
29483 Target.setIsRenamable(false);
29484
29485 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: RISCV::KCFI_CHECK))
29486 .addReg(RegNo: Target.getReg())
29487 .addImm(Val: MBBI->getCFIType())
29488 .getInstr();
29489}
29490
29491#define GET_REGISTER_MATCHER
29492#include "RISCVGenAsmMatcher.inc"
29493
29494Register
29495RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
29496 const MachineFunction &MF) const {
29497 Register Reg = MatchRegisterAltName(Name: RegName);
29498 if (!Reg)
29499 Reg = MatchRegisterName(Name: RegName);
29500 if (!Reg)
29501 return Reg;
29502
29503 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
29504 if (!ReservedRegs.test(Idx: Reg) && !Subtarget.isRegisterReservedByUser(i: Reg))
29505 reportFatalUsageError(reason: Twine("Trying to obtain non-reserved register \"" +
29506 StringRef(RegName) + "\"."));
29507 return Reg;
29508}
29509
29510MachineMemOperand::Flags
29511RISCVTargetLowering::getTargetMMOFlags(const Instruction &I) const {
29512 const MDNode *NontemporalInfo = I.getMetadata(KindID: LLVMContext::MD_nontemporal);
29513
29514 if (NontemporalInfo == nullptr)
29515 return MachineMemOperand::MONone;
29516
29517 // 1 for default value work as __RISCV_NTLH_ALL
29518 // 2 -> __RISCV_NTLH_INNERMOST_PRIVATE
29519 // 3 -> __RISCV_NTLH_ALL_PRIVATE
29520 // 4 -> __RISCV_NTLH_INNERMOST_SHARED
29521 // 5 -> __RISCV_NTLH_ALL
29522 int NontemporalLevel = 5;
29523 const MDNode *RISCVNontemporalInfo =
29524 I.getMetadata(Kind: "riscv-nontemporal-domain");
29525 if (RISCVNontemporalInfo != nullptr)
29526 NontemporalLevel =
29527 cast<ConstantInt>(
29528 Val: cast<ConstantAsMetadata>(Val: RISCVNontemporalInfo->getOperand(I: 0))
29529 ->getValue())
29530 ->getZExtValue();
29531
29532 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
29533 "RISC-V target doesn't support this non-temporal domain.");
29534
29535 NontemporalLevel -= 2;
29536 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
29537 if (NontemporalLevel & 0b1)
29538 Flags |= MONontemporalBit0;
29539 if (NontemporalLevel & 0b10)
29540 Flags |= MONontemporalBit1;
29541
29542 return Flags;
29543}
29544
29545MachineMemOperand::Flags
29546RISCVTargetLowering::getTargetMMOFlags(const MemSDNode &Node) const {
29547
29548 MachineMemOperand::Flags NodeFlags = Node.getMemOperand()->getFlags();
29549 MachineMemOperand::Flags TargetFlags = MachineMemOperand::MONone;
29550 TargetFlags |= (NodeFlags & MONontemporalBit0);
29551 TargetFlags |= (NodeFlags & MONontemporalBit1);
29552 return TargetFlags;
29553}
29554
29555bool RISCVTargetLowering::areTwoSDNodeTargetMMOFlagsMergeable(
29556 const MemSDNode &NodeX, const MemSDNode &NodeY) const {
29557 return getTargetMMOFlags(Node: NodeX) == getTargetMMOFlags(Node: NodeY);
29558}
29559
29560bool RISCVTargetLowering::isCtpopFast(EVT VT) const {
29561 if (VT.isVector()) {
29562 EVT SVT = VT.getVectorElementType();
29563 // If the element type is legal we can use cpop.v if it is enabled.
29564 if (isLegalElementTypeForRVV(ScalarTy: SVT))
29565 return Subtarget.hasStdExtZvbb();
29566 // Don't consider it fast if the type needs to be legalized or scalarized.
29567 return false;
29568 }
29569
29570 return Subtarget.hasCPOPLike() && (VT == MVT::i32 || VT == MVT::i64);
29571}
29572
29573unsigned RISCVTargetLowering::getCustomCtpopCost(EVT VT,
29574 ISD::CondCode Cond) const {
29575 return isCtpopFast(VT) ? 0 : 1;
29576}
29577
29578bool RISCVTargetLowering::shouldInsertFencesForAtomic(
29579 const Instruction *I) const {
29580 if (Subtarget.hasStdExtZalasr()) {
29581 if (Subtarget.hasStdExtZtso()) {
29582 // Zalasr + TSO means that atomic_load_acquire and atomic_store_release
29583 // should be lowered to plain load/store. The easiest way to do this is
29584 // to say we should insert fences for them, and the fence insertion code
29585 // will just not insert any fences
29586 auto *LI = dyn_cast<LoadInst>(Val: I);
29587 auto *SI = dyn_cast<StoreInst>(Val: I);
29588 if ((LI &&
29589 (LI->getOrdering() == AtomicOrdering::SequentiallyConsistent)) ||
29590 (SI &&
29591 (SI->getOrdering() == AtomicOrdering::SequentiallyConsistent))) {
29592 // Here, this is a load or store which is seq_cst, and needs a .aq or
29593 // .rl therefore we shouldn't try to insert fences
29594 return false;
29595 }
29596 // Here, we are a TSO inst that isn't a seq_cst load/store
29597 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
29598 }
29599 return false;
29600 }
29601 // Note that one specific case requires fence insertion for an
29602 // AtomicCmpXchgInst but is handled via the RISCVZacasABIFix pass rather
29603 // than this hook due to limitations in the interface here.
29604 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
29605}
29606
29607bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
29608
29609 // GISel support is in progress or complete for these opcodes.
29610 unsigned Op = Inst.getOpcode();
29611 if (Op == Instruction::Add || Op == Instruction::Sub ||
29612 Op == Instruction::And || Op == Instruction::Or ||
29613 Op == Instruction::Xor || Op == Instruction::InsertElement ||
29614 Op == Instruction::ShuffleVector || Op == Instruction::Load ||
29615 Op == Instruction::Freeze || Op == Instruction::Store)
29616 return false;
29617
29618 if (auto *II = dyn_cast<IntrinsicInst>(Val: &Inst)) {
29619 // Mark RVV intrinsic as supported.
29620 if (RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: II->getIntrinsicID())) {
29621 // GISel doesn't support tuple types yet. It also doesn't suport returning
29622 // a struct containing a scalable vector like vleff.
29623 if (Inst.getType()->isRISCVVectorTupleTy() ||
29624 Inst.getType()->isStructTy())
29625 return true;
29626
29627 for (unsigned i = 0; i < II->arg_size(); ++i)
29628 if (II->getArgOperand(i)->getType()->isRISCVVectorTupleTy())
29629 return true;
29630
29631 return false;
29632 }
29633 if (II->getIntrinsicID() == Intrinsic::vector_extract ||
29634 II->getIntrinsicID() == Intrinsic::vector_insert)
29635 return false;
29636 }
29637
29638 if (Inst.getType()->isScalableTy())
29639 return true;
29640
29641 for (unsigned i = 0; i < Inst.getNumOperands(); ++i)
29642 if (Inst.getOperand(i)->getType()->isScalableTy() &&
29643 !isa<ReturnInst>(Val: &Inst))
29644 return true;
29645
29646 return false;
29647}
29648
29649SDValue
29650RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
29651 SelectionDAG &DAG,
29652 SmallVectorImpl<SDNode *> &Created) const {
29653 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
29654 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
29655 return SDValue(N, 0); // Lower SDIV as SDIV
29656
29657 // Only perform this transform if short forward branch opt is supported.
29658 if (!Subtarget.hasShortForwardBranchIALU())
29659 return SDValue();
29660 EVT VT = N->getValueType(ResNo: 0);
29661 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit())))
29662 return SDValue();
29663
29664 // Ensure 2**k-1 < 2048 so that we can just emit a single addi/addiw.
29665 if (Divisor.sgt(RHS: 2048) || Divisor.slt(RHS: -2048))
29666 return SDValue();
29667 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
29668}
29669
29670bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
29671 EVT VT, const APInt &AndMask) const {
29672 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXTHeadCondMov())
29673 return !Subtarget.hasBEXTILike() && AndMask.ugt(RHS: 1024);
29674 return TargetLowering::shouldFoldSelectWithSingleBitTest(VT, AndMask);
29675}
29676
29677unsigned RISCVTargetLowering::getMinimumJumpTableEntries() const {
29678 return Subtarget.getMinimumJumpTableEntries();
29679}
29680
29681SDValue RISCVTargetLowering::expandIndirectJTBranch(const SDLoc &dl,
29682 SDValue Value, SDValue Addr,
29683 int JTI,
29684 SelectionDAG &DAG) const {
29685 const MachineFunction &MF = DAG.getMachineFunction();
29686 if (MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch()) {
29687 // When cf-protection-branch enabled, we need to use software guarded
29688 // branch for jump table branch.
29689 SDValue Chain = Value;
29690 // Jump table debug info is only needed if CodeView is enabled.
29691 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF())
29692 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
29693 return DAG.getNode(Opcode: RISCVISD::SW_GUARDED_BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
29694 }
29695 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
29696}
29697
29698// If an output pattern produces multiple instructions tablegen may pick an
29699// arbitrary type from an instructions destination register class to use for the
29700// VT of that MachineSDNode. This VT may be used to look up the representative
29701// register class. If the type isn't legal, the default implementation will
29702// not find a register class.
29703//
29704// Some integer types smaller than XLen are listed in the GPR register class to
29705// support isel patterns for GISel, but are not legal in SelectionDAG. The
29706// arbitrary type tablegen picks may be one of these smaller types.
29707//
29708// f16 and bf16 are both valid for the FPR16 or GPRF16 register class. It's
29709// possible for tablegen to pick bf16 as the arbitrary type for an f16 pattern.
29710std::pair<const TargetRegisterClass *, uint8_t>
29711RISCVTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
29712 MVT VT) const {
29713 switch (VT.SimpleTy) {
29714 default:
29715 break;
29716 case MVT::i8:
29717 case MVT::i16:
29718 case MVT::i32:
29719 return TargetLowering::findRepresentativeClass(TRI, VT: Subtarget.getXLenVT());
29720 case MVT::bf16:
29721 case MVT::f16:
29722 return TargetLowering::findRepresentativeClass(TRI, VT: MVT::f32);
29723 }
29724
29725 return TargetLowering::findRepresentativeClass(TRI, VT);
29726}
29727
29728namespace llvm::RISCVVIntrinsicsTable {
29729
29730#define GET_RISCVVIntrinsicsTable_IMPL
29731#include "RISCVGenSearchableTables.inc"
29732
29733} // namespace llvm::RISCVVIntrinsicsTable
29734
29735bool RISCVTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
29736
29737 // If the function specifically requests inline stack probes, emit them.
29738 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
29739 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
29740 "inline-asm";
29741
29742 return false;
29743}
29744
29745unsigned RISCVTargetLowering::getStackProbeSize(const MachineFunction &MF,
29746 Align StackAlign) const {
29747 // The default stack probe size is 4096 if the function has no
29748 // stack-probe-size attribute.
29749 const Function &Fn = MF.getFunction();
29750 unsigned StackProbeSize =
29751 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
29752 // Round down to the stack alignment.
29753 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
29754 return StackProbeSize ? StackProbeSize : StackAlign.value();
29755}
29756
29757SDValue RISCVTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
29758 SelectionDAG &DAG) const {
29759 MachineFunction &MF = DAG.getMachineFunction();
29760 if (!hasInlineStackProbe(MF))
29761 return SDValue();
29762
29763 MVT XLenVT = Subtarget.getXLenVT();
29764 // Get the inputs.
29765 SDValue Chain = Op.getOperand(i: 0);
29766 SDValue Size = Op.getOperand(i: 1);
29767
29768 MaybeAlign Align =
29769 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
29770 SDLoc dl(Op);
29771 EVT VT = Op.getValueType();
29772
29773 // Construct the new SP value in a GPR.
29774 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: RISCV::X2, VT: XLenVT);
29775 Chain = SP.getValue(R: 1);
29776 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: XLenVT, N1: SP, N2: Size);
29777 if (Align)
29778 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
29779 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
29780
29781 // Set the real SP to the new value with a probing loop.
29782 Chain = DAG.getNode(Opcode: RISCVISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
29783 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
29784}
29785
29786MachineBasicBlock *
29787RISCVTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
29788 MachineBasicBlock *MBB) const {
29789 MachineFunction &MF = *MBB->getParent();
29790 MachineBasicBlock::iterator MBBI = MI.getIterator();
29791 DebugLoc DL = MBB->findDebugLoc(MBBI);
29792 Register TargetReg = MI.getOperand(i: 0).getReg();
29793
29794 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
29795 bool IsRV64 = Subtarget.is64Bit();
29796 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
29797 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
29798 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
29799
29800 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
29801 MachineBasicBlock *LoopTestMBB =
29802 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
29803 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
29804 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
29805 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
29806 Register SPReg = RISCV::X2;
29807 Register ScratchReg =
29808 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
29809
29810 // ScratchReg = ProbeSize
29811 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
29812
29813 // LoopTest:
29814 // SUB SP, SP, ProbeSize
29815 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
29816 .addReg(RegNo: SPReg)
29817 .addReg(RegNo: ScratchReg);
29818
29819 // s[d|w] zero, 0(sp)
29820 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
29821 MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
29822 .addReg(RegNo: RISCV::X0)
29823 .addReg(RegNo: SPReg)
29824 .addImm(Val: 0);
29825
29826 // BLTU TargetReg, SP, LoopTest
29827 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BLTU))
29828 .addReg(RegNo: TargetReg)
29829 .addReg(RegNo: SPReg)
29830 .addMBB(MBB: LoopTestMBB);
29831
29832 // Adjust with: MV SP, TargetReg.
29833 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: SPReg)
29834 .addReg(RegNo: TargetReg)
29835 .addImm(Val: 0);
29836
29837 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
29838 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
29839
29840 LoopTestMBB->addSuccessor(Succ: ExitMBB);
29841 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
29842 MBB->addSuccessor(Succ: LoopTestMBB);
29843
29844 MI.eraseFromParent();
29845 MF.getInfo<RISCVMachineFunctionInfo>()->setDynamicAllocation();
29846 return ExitMBB->begin()->getParent();
29847}
29848
29849ArrayRef<MCPhysReg> RISCVTargetLowering::getRoundingControlRegisters() const {
29850 if (Subtarget.hasStdExtFOrZfinx()) {
29851 static const MCPhysReg RCRegs[] = {RISCV::FRM, RISCV::FFLAGS};
29852 return RCRegs;
29853 }
29854 return {};
29855}
29856
29857bool RISCVTargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
29858 EVT VT = Y.getValueType();
29859
29860 if (VT.isVector())
29861 return false;
29862
29863 return VT.getSizeInBits() <= Subtarget.getXLen();
29864}
29865
29866bool RISCVTargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
29867 SDValue N1) const {
29868 if (!N0.hasOneUse())
29869 return false;
29870
29871 // Avoid reassociating expressions that can be lowered to vector
29872 // multiply accumulate (i.e. add (mul x, y), z)
29873 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::MUL &&
29874 (N0.getValueType().isVector() && Subtarget.hasVInstructions()))
29875 return false;
29876
29877 return true;
29878}
29879