1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "MCTargetDesc/RISCVMatInt.h"
17#include "RISCVISelLowering.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSelectionDAGInfo.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/SDPatternMatch.h"
22#include "llvm/IR/IntrinsicsRISCV.h"
23#include "llvm/Support/Alignment.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/MathExtras.h"
26#include "llvm/Support/raw_ostream.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "riscv-isel"
31#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
32
33extern cl::opt<uint32_t> PreferredLandingPadLabel;
34
35static cl::opt<bool> UsePseudoMovImm(
36 "riscv-use-rematerializable-movimm", cl::Hidden,
37 cl::desc("Use a rematerializable pseudoinstruction for 2 instruction "
38 "constant materialization"),
39 cl::init(Val: false));
40
41#define GET_DAGISEL_BODY RISCVDAGToDAGISel
42#include "RISCVGenDAGISel.inc"
43
44void RISCVDAGToDAGISel::PreprocessISelDAG() {
45 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
46
47 bool MadeChange = false;
48 while (Position != CurDAG->allnodes_begin()) {
49 SDNode *N = &*--Position;
50 if (N->use_empty())
51 continue;
52
53 SDValue Result;
54 switch (N->getOpcode()) {
55 case ISD::SPLAT_VECTOR: {
56 if (Subtarget->hasStdExtP())
57 break;
58 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
59 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
60 MVT VT = N->getSimpleValueType(ResNo: 0);
61 unsigned Opc =
62 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
63 SDLoc DL(N);
64 SDValue VL = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
65 SDValue Src = N->getOperand(Num: 0);
66 if (VT.isInteger())
67 Src = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget->getXLenVT(),
68 Operand: N->getOperand(Num: 0));
69 Result = CurDAG->getNode(Opcode: Opc, DL, VT, N1: CurDAG->getUNDEF(VT), N2: Src, N3: VL);
70 break;
71 }
72 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
73 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
74 // load. Done after lowering and combining so that we have a chance to
75 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
76 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
77 MVT VT = N->getSimpleValueType(ResNo: 0);
78 SDValue Passthru = N->getOperand(Num: 0);
79 SDValue Lo = N->getOperand(Num: 1);
80 SDValue Hi = N->getOperand(Num: 2);
81 SDValue VL = N->getOperand(Num: 3);
82 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
83 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
84 "Unexpected VTs!");
85 MachineFunction &MF = CurDAG->getMachineFunction();
86 SDLoc DL(N);
87
88 // Create temporary stack for each expanding node.
89 SDValue StackSlot =
90 CurDAG->CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 8), Alignment: Align(8));
91 int FI = cast<FrameIndexSDNode>(Val: StackSlot.getNode())->getIndex();
92 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
93
94 SDValue Chain = CurDAG->getEntryNode();
95 Lo = CurDAG->getStore(Chain, dl: DL, Val: Lo, Ptr: StackSlot, PtrInfo: MPI, Alignment: Align(8));
96
97 SDValue OffsetSlot =
98 CurDAG->getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL);
99 Hi = CurDAG->getStore(Chain, dl: DL, Val: Hi, Ptr: OffsetSlot, PtrInfo: MPI.getWithOffset(O: 4),
100 Alignment: Align(8));
101
102 Chain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
103
104 SDVTList VTs = CurDAG->getVTList(VTs: {VT, MVT::Other});
105 SDValue IntID =
106 CurDAG->getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: MVT::i64);
107 SDValue Ops[] = {Chain,
108 IntID,
109 Passthru,
110 StackSlot,
111 CurDAG->getRegister(Reg: RISCV::X0, VT: MVT::i64),
112 VL};
113
114 Result = CurDAG->getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
115 MemVT: MVT::i64, PtrInfo: MPI, Alignment: Align(8),
116 Flags: MachineMemOperand::MOLoad);
117 break;
118 }
119 case ISD::FP_EXTEND: {
120 // We only have vector patterns for riscv_fpextend_vl in isel.
121 SDLoc DL(N);
122 MVT VT = N->getSimpleValueType(ResNo: 0);
123 if (!VT.isVector())
124 break;
125 SDValue VLMAX = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
126 SDValue TrueMask = CurDAG->getNode(
127 Opcode: RISCVISD::VMSET_VL, DL, VT: VT.changeVectorElementType(EltVT: MVT::i1), Operand: VLMAX);
128 Result = CurDAG->getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT, N1: N->getOperand(Num: 0),
129 N2: TrueMask, N3: VLMAX);
130 break;
131 }
132 }
133
134 if (Result) {
135 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
136 LLVM_DEBUG(N->dump(CurDAG));
137 LLVM_DEBUG(dbgs() << "\nNew: ");
138 LLVM_DEBUG(Result->dump(CurDAG));
139 LLVM_DEBUG(dbgs() << "\n");
140
141 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
142 MadeChange = true;
143 }
144 }
145
146 if (MadeChange)
147 CurDAG->RemoveDeadNodes();
148}
149
150void RISCVDAGToDAGISel::PostprocessISelDAG() {
151 HandleSDNode Dummy(CurDAG->getRoot());
152 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
153
154 bool MadeChange = false;
155 while (Position != CurDAG->allnodes_begin()) {
156 SDNode *N = &*--Position;
157 // Skip dead nodes and any non-machine opcodes.
158 if (N->use_empty() || !N->isMachineOpcode())
159 continue;
160
161 MadeChange |= doPeepholeSExtW(Node: N);
162
163 // FIXME: This is here only because the VMerge transform doesn't
164 // know how to handle masked true inputs. Once that has been moved
165 // to post-ISEL, this can be deleted as well.
166 MadeChange |= doPeepholeMaskedRVV(Node: cast<MachineSDNode>(Val: N));
167 }
168
169 CurDAG->setRoot(Dummy.getValue());
170
171 // After we're done with everything else, convert IMPLICIT_DEF
172 // passthru operands to NoRegister. This is required to workaround
173 // an optimization deficiency in MachineCSE. This really should
174 // be merged back into each of the patterns (i.e. there's no good
175 // reason not to go directly to NoReg), but is being done this way
176 // to allow easy backporting.
177 MadeChange |= doPeepholeNoRegPassThru();
178
179 if (MadeChange)
180 CurDAG->RemoveDeadNodes();
181}
182
183static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
184 RISCVMatInt::InstSeq &Seq) {
185 SDValue SrcReg = CurDAG->getRegister(Reg: RISCV::X0, VT);
186 for (const RISCVMatInt::Inst &Inst : Seq) {
187 SDValue SDImm = CurDAG->getSignedTargetConstant(Val: Inst.getImm(), DL, VT);
188 SDNode *Result = nullptr;
189 switch (Inst.getOpndKind()) {
190 case RISCVMatInt::Imm:
191 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SDImm);
192 break;
193 case RISCVMatInt::RegX0:
194 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg,
195 Op2: CurDAG->getRegister(Reg: RISCV::X0, VT));
196 break;
197 case RISCVMatInt::RegReg:
198 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SrcReg);
199 break;
200 case RISCVMatInt::RegImm:
201 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SDImm);
202 break;
203 }
204
205 // Only the first instruction has X0 as its source.
206 SrcReg = SDValue(Result, 0);
207 }
208
209 return SrcReg;
210}
211
212static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
213 int64_t Imm, const RISCVSubtarget &Subtarget) {
214 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
215
216 // Use a rematerializable pseudo instruction for short sequences if enabled.
217 if (Seq.size() == 2 && UsePseudoMovImm)
218 return SDValue(
219 CurDAG->getMachineNode(Opcode: RISCV::PseudoMovImm, dl: DL, VT,
220 Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT)),
221 0);
222
223 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
224 // worst an LUI+ADDIW. This will require an extra register, but avoids a
225 // constant pool.
226 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
227 // low and high 32 bits are the same and bit 31 and 63 are set.
228 if (Seq.size() > 3) {
229 unsigned ShiftAmt, AddOpc;
230 RISCVMatInt::InstSeq SeqLo =
231 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
232 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
233 SDValue Lo = selectImmSeq(CurDAG, DL, VT, Seq&: SeqLo);
234
235 SDValue SLLI = SDValue(
236 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: Lo,
237 Op2: CurDAG->getTargetConstant(Val: ShiftAmt, DL, VT)),
238 0);
239 return SDValue(CurDAG->getMachineNode(Opcode: AddOpc, dl: DL, VT, Op1: Lo, Op2: SLLI), 0);
240 }
241 }
242
243 // Otherwise, use the original sequence.
244 return selectImmSeq(CurDAG, DL, VT, Seq);
245}
246
247void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
248 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
249 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
250 bool IsLoad, MVT *IndexVT) {
251 SDValue Chain = Node->getOperand(Num: 0);
252
253 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Base pointer.
254
255 if (IsStridedOrIndexed) {
256 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Index.
257 if (IndexVT)
258 *IndexVT = Operands.back()->getSimpleValueType(ResNo: 0);
259 }
260
261 if (IsMasked) {
262 SDValue Mask = Node->getOperand(Num: CurOp++);
263 Operands.push_back(Elt: Mask);
264 }
265 SDValue VL;
266 selectVLOp(N: Node->getOperand(Num: CurOp++), VL);
267 Operands.push_back(Elt: VL);
268
269 MVT XLenVT = Subtarget->getXLenVT();
270 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
271 Operands.push_back(Elt: SEWOp);
272
273 // At the IR layer, all the masked load intrinsics have policy operands,
274 // none of the others do. All have passthru operands. For our pseudos,
275 // all loads have policy operands.
276 if (IsLoad) {
277 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
278 if (IsMasked)
279 Policy = Node->getConstantOperandVal(Num: CurOp++);
280 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
281 Operands.push_back(Elt: PolicyOp);
282 }
283
284 Operands.push_back(Elt: Chain); // Chain.
285}
286
287void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked,
288 bool IsStrided) {
289 SDLoc DL(Node);
290 MVT VT = Node->getSimpleValueType(ResNo: 0);
291 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
292 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
293
294 unsigned CurOp = 2;
295 SmallVector<SDValue, 8> Operands;
296
297 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
298
299 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
300 Operands, /*IsLoad=*/true);
301
302 const RISCV::VLSEGPseudo *P =
303 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
304 LMUL: static_cast<unsigned>(LMUL));
305 MachineSDNode *Load =
306 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
307
308 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
309
310 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
311 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
312 CurDAG->RemoveDeadNode(N: Node);
313}
314
315void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, unsigned NF,
316 bool IsMasked) {
317 SDLoc DL(Node);
318 MVT VT = Node->getSimpleValueType(ResNo: 0);
319 MVT XLenVT = Subtarget->getXLenVT();
320 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
321 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
322
323 unsigned CurOp = 2;
324 SmallVector<SDValue, 7> Operands;
325
326 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
327
328 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
329 /*IsStridedOrIndexed*/ false, Operands,
330 /*IsLoad=*/true);
331
332 const RISCV::VLSEGPseudo *P =
333 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
334 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
335 MachineSDNode *Load = CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped,
336 VT2: XLenVT, VT3: MVT::Other, Ops: Operands);
337
338 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
339
340 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0)); // Result
341 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1)); // VL
342 ReplaceUses(F: SDValue(Node, 2), T: SDValue(Load, 2)); // Chain
343 CurDAG->RemoveDeadNode(N: Node);
344}
345
346void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked,
347 bool IsOrdered) {
348 SDLoc DL(Node);
349 MVT VT = Node->getSimpleValueType(ResNo: 0);
350 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
351 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
352
353 unsigned CurOp = 2;
354 SmallVector<SDValue, 8> Operands;
355
356 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
357
358 MVT IndexVT;
359 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
360 /*IsStridedOrIndexed*/ true, Operands,
361 /*IsLoad=*/true, IndexVT: &IndexVT);
362
363#ifndef NDEBUG
364 // Number of element = RVVBitsPerBlock * LMUL / SEW
365 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
366 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
367 if (DecodedLMUL.second)
368 ContainedTyNumElts /= DecodedLMUL.first;
369 else
370 ContainedTyNumElts *= DecodedLMUL.first;
371 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
372 "Element count mismatch");
373#endif
374
375 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
376 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
377 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
378 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
379 "values when XLEN=32");
380 }
381 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
382 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
383 IndexLMUL: static_cast<unsigned>(IndexLMUL));
384 MachineSDNode *Load =
385 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
386
387 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
388
389 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
390 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
391 CurDAG->RemoveDeadNode(N: Node);
392}
393
394void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked,
395 bool IsStrided) {
396 SDLoc DL(Node);
397 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
398 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
399 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
400
401 unsigned CurOp = 2;
402 SmallVector<SDValue, 8> Operands;
403
404 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
405
406 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
407 Operands);
408
409 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
410 NF, Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
411 MachineSDNode *Store =
412 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
413
414 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
415
416 ReplaceNode(F: Node, T: Store);
417}
418
419void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked,
420 bool IsOrdered) {
421 SDLoc DL(Node);
422 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
423 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
424 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
425
426 unsigned CurOp = 2;
427 SmallVector<SDValue, 8> Operands;
428
429 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
430
431 MVT IndexVT;
432 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
433 /*IsStridedOrIndexed*/ true, Operands,
434 /*IsLoad=*/false, IndexVT: &IndexVT);
435
436#ifndef NDEBUG
437 // Number of element = RVVBitsPerBlock * LMUL / SEW
438 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
439 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
440 if (DecodedLMUL.second)
441 ContainedTyNumElts /= DecodedLMUL.first;
442 else
443 ContainedTyNumElts *= DecodedLMUL.first;
444 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
445 "Element count mismatch");
446#endif
447
448 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
449 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
450 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
451 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
452 "values when XLEN=32");
453 }
454 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
455 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
456 IndexLMUL: static_cast<unsigned>(IndexLMUL));
457 MachineSDNode *Store =
458 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
459
460 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
461
462 ReplaceNode(F: Node, T: Store);
463}
464
465void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) {
466 if (!Subtarget->hasVInstructions())
467 return;
468
469 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
470
471 SDLoc DL(Node);
472 MVT XLenVT = Subtarget->getXLenVT();
473
474 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
475
476 assert((IntNo == Intrinsic::riscv_vsetvli ||
477 IntNo == Intrinsic::riscv_vsetvlimax) &&
478 "Unexpected vsetvli intrinsic");
479
480 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
481 unsigned Offset = (VLMax ? 1 : 2);
482
483 assert(Node->getNumOperands() == Offset + 2 &&
484 "Unexpected number of operands");
485
486 unsigned SEW =
487 RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: Offset) & 0x7);
488 RISCVVType::VLMUL VLMul = static_cast<RISCVVType::VLMUL>(
489 Node->getConstantOperandVal(Num: Offset + 1) & 0x7);
490
491 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
492 /*MaskAgnostic*/ true);
493 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
494
495 SDValue VLOperand;
496 unsigned Opcode = RISCV::PseudoVSETVLI;
497 if (auto *C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
498 if (auto VLEN = Subtarget->getRealVLen())
499 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
500 VLMax = true;
501 }
502 if (VLMax || isAllOnesConstant(V: Node->getOperand(Num: 1))) {
503 VLOperand = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
504 Opcode = RISCV::PseudoVSETVLIX0;
505 } else {
506 VLOperand = Node->getOperand(Num: 1);
507
508 if (auto *C = dyn_cast<ConstantSDNode>(Val&: VLOperand)) {
509 uint64_t AVL = C->getZExtValue();
510 if (isUInt<5>(x: AVL)) {
511 SDValue VLImm = CurDAG->getTargetConstant(Val: AVL, DL, VT: XLenVT);
512 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: RISCV::PseudoVSETIVLI, dl: DL,
513 VT: XLenVT, Op1: VLImm, Op2: VTypeIOp));
514 return;
515 }
516 }
517 }
518
519 ReplaceNode(F: Node,
520 T: CurDAG->getMachineNode(Opcode, dl: DL, VT: XLenVT, Op1: VLOperand, Op2: VTypeIOp));
521}
522
523void RISCVDAGToDAGISel::selectXSfmmVSET(SDNode *Node) {
524 if (!Subtarget->hasVendorXSfmmbase())
525 return;
526
527 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
528
529 SDLoc DL(Node);
530 MVT XLenVT = Subtarget->getXLenVT();
531
532 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
533
534 assert((IntNo == Intrinsic::riscv_sf_vsettnt ||
535 IntNo == Intrinsic::riscv_sf_vsettm ||
536 IntNo == Intrinsic::riscv_sf_vsettk) &&
537 "Unexpected XSfmm vset intrinsic");
538
539 unsigned SEW = RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: 2));
540 unsigned Widen = RISCVVType::decodeTWiden(TWiden: Node->getConstantOperandVal(Num: 3));
541 unsigned PseudoOpCode =
542 IntNo == Intrinsic::riscv_sf_vsettnt ? RISCV::PseudoSF_VSETTNT
543 : IntNo == Intrinsic::riscv_sf_vsettm ? RISCV::PseudoSF_VSETTM
544 : RISCV::PseudoSF_VSETTK;
545
546 if (IntNo == Intrinsic::riscv_sf_vsettnt) {
547 unsigned VTypeI = RISCVVType::encodeXSfmmVType(SEW, Widen, AltFmt: 0);
548 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
549
550 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
551 Op1: Node->getOperand(Num: 1), Op2: VTypeIOp));
552 } else {
553 SDValue Log2SEW = CurDAG->getTargetConstant(Val: Log2_32(Value: SEW), DL, VT: XLenVT);
554 SDValue TWiden = CurDAG->getTargetConstant(Val: Widen, DL, VT: XLenVT);
555 ReplaceNode(F: Node,
556 T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
557 Op1: Node->getOperand(Num: 1), Op2: Log2SEW, Op3: TWiden));
558 }
559}
560
561bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
562 MVT VT = Node->getSimpleValueType(ResNo: 0);
563 unsigned Opcode = Node->getOpcode();
564 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
565 "Unexpected opcode");
566 SDLoc DL(Node);
567
568 // For operations of the form (x << C1) op C2, check if we can use
569 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
570 SDValue N0 = Node->getOperand(Num: 0);
571 SDValue N1 = Node->getOperand(Num: 1);
572
573 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
574 if (!Cst)
575 return false;
576
577 int64_t Val = Cst->getSExtValue();
578
579 // Check if immediate can already use ANDI/ORI/XORI.
580 if (isInt<12>(x: Val))
581 return false;
582
583 SDValue Shift = N0;
584
585 // If Val is simm32 and we have a sext_inreg from i32, then the binop
586 // produces at least 33 sign bits. We can peek through the sext_inreg and use
587 // a SLLIW at the end.
588 bool SignExt = false;
589 if (isInt<32>(x: Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
590 N0.hasOneUse() && cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT() == MVT::i32) {
591 SignExt = true;
592 Shift = N0.getOperand(i: 0);
593 }
594
595 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
596 return false;
597
598 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1));
599 if (!ShlCst)
600 return false;
601
602 uint64_t ShAmt = ShlCst->getZExtValue();
603
604 // Make sure that we don't change the operation by removing bits.
605 // This only matters for OR and XOR, AND is unaffected.
606 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(N: ShAmt);
607 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
608 return false;
609
610 int64_t ShiftedVal = Val >> ShAmt;
611 if (!isInt<12>(x: ShiftedVal))
612 return false;
613
614 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
615 if (SignExt && ShAmt >= 32)
616 return false;
617
618 // Ok, we can reorder to get a smaller immediate.
619 unsigned BinOpc;
620 switch (Opcode) {
621 default: llvm_unreachable("Unexpected opcode");
622 case ISD::AND: BinOpc = RISCV::ANDI; break;
623 case ISD::OR: BinOpc = RISCV::ORI; break;
624 case ISD::XOR: BinOpc = RISCV::XORI; break;
625 }
626
627 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
628
629 SDNode *BinOp = CurDAG->getMachineNode(
630 Opcode: BinOpc, dl: DL, VT, Op1: Shift.getOperand(i: 0),
631 Op2: CurDAG->getSignedTargetConstant(Val: ShiftedVal, DL, VT));
632 SDNode *SLLI =
633 CurDAG->getMachineNode(Opcode: ShOpc, dl: DL, VT, Op1: SDValue(BinOp, 0),
634 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
635 ReplaceNode(F: Node, T: SLLI);
636 return true;
637}
638
639bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
640 unsigned Opc;
641
642 if (Subtarget->hasVendorXTHeadBb())
643 Opc = RISCV::TH_EXT;
644 else if (Subtarget->hasVendorXAndesPerf())
645 Opc = RISCV::NDS_BFOS;
646 else if (Subtarget->hasVendorXqcibm())
647 Opc = RISCV::QC_EXT;
648 else
649 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
650 return false;
651
652 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
653 if (!N1C)
654 return false;
655
656 SDValue N0 = Node->getOperand(Num: 0);
657 if (!N0.hasOneUse())
658 return false;
659
660 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
661 const SDLoc &DL, MVT VT) {
662 if (Opc == RISCV::QC_EXT) {
663 // QC.EXT X, width, shamt
664 // shamt is the same as Lsb
665 // width is the number of bits to extract from the Lsb
666 Msb = Msb - Lsb + 1;
667 }
668 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
669 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
670 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
671 };
672
673 SDLoc DL(Node);
674 MVT VT = Node->getSimpleValueType(ResNo: 0);
675 const unsigned RightShAmt = N1C->getZExtValue();
676
677 // Transform (sra (shl X, C1) C2) with C1 < C2
678 // -> (SignedBitfieldExtract X, msb, lsb)
679 if (N0.getOpcode() == ISD::SHL) {
680 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
681 if (!N01C)
682 return false;
683
684 const unsigned LeftShAmt = N01C->getZExtValue();
685 // Make sure that this is a bitfield extraction (i.e., the shift-right
686 // amount can not be less than the left-shift).
687 if (LeftShAmt > RightShAmt)
688 return false;
689
690 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
691 const unsigned Msb = MsbPlusOne - 1;
692 const unsigned Lsb = RightShAmt - LeftShAmt;
693
694 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
695 ReplaceNode(F: Node, T: Sbe);
696 return true;
697 }
698
699 // Transform (sra (sext_inreg X, _), C) ->
700 // (SignedBitfieldExtract X, msb, lsb)
701 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
702 unsigned ExtSize =
703 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
704
705 // ExtSize of 32 should use sraiw via tablegen pattern.
706 if (ExtSize == 32)
707 return false;
708
709 const unsigned Msb = ExtSize - 1;
710 // If the shift-right amount is greater than Msb, it means that extracts
711 // the X[Msb] bit and sign-extend it.
712 const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
713
714 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
715 ReplaceNode(F: Node, T: Sbe);
716 return true;
717 }
718
719 return false;
720}
721
722bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
723 // Only supported with XAndesPerf at the moment.
724 if (!Subtarget->hasVendorXAndesPerf())
725 return false;
726
727 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
728 if (!N1C)
729 return false;
730
731 SDValue N0 = Node->getOperand(Num: 0);
732 if (!N0.hasOneUse())
733 return false;
734
735 auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
736 const SDLoc &DL, MVT VT) {
737 unsigned Opc = RISCV::NDS_BFOS;
738 // If the Lsb is equal to the Msb, then the Lsb should be 0.
739 if (Lsb == Msb)
740 Lsb = 0;
741 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
742 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
743 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
744 };
745
746 SDLoc DL(Node);
747 MVT VT = Node->getSimpleValueType(ResNo: 0);
748 const unsigned RightShAmt = N1C->getZExtValue();
749
750 // Transform (sra (shl X, C1) C2) with C1 > C2
751 // -> (NDS.BFOS X, lsb, msb)
752 if (N0.getOpcode() == ISD::SHL) {
753 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
754 if (!N01C)
755 return false;
756
757 const unsigned LeftShAmt = N01C->getZExtValue();
758 // Make sure that this is a bitfield insertion (i.e., the shift-right
759 // amount should be less than the left-shift).
760 if (LeftShAmt <= RightShAmt)
761 return false;
762
763 const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
764 const unsigned Msb = MsbPlusOne - 1;
765 const unsigned Lsb = LeftShAmt - RightShAmt;
766
767 SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
768 ReplaceNode(F: Node, T: Sbi);
769 return true;
770 }
771
772 return false;
773}
774
775bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
776 const SDLoc &DL, MVT VT,
777 SDValue X, unsigned Msb,
778 unsigned Lsb) {
779 unsigned Opc;
780
781 if (Subtarget->hasVendorXTHeadBb()) {
782 Opc = RISCV::TH_EXTU;
783 } else if (Subtarget->hasVendorXAndesPerf()) {
784 Opc = RISCV::NDS_BFOZ;
785 } else if (Subtarget->hasVendorXqcibm()) {
786 Opc = RISCV::QC_EXTU;
787 // QC.EXTU X, width, shamt
788 // shamt is the same as Lsb
789 // width is the number of bits to extract from the Lsb
790 Msb = Msb - Lsb + 1;
791 } else {
792 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
793 return false;
794 }
795
796 SDNode *Ube = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
797 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
798 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
799 ReplaceNode(F: Node, T: Ube);
800 return true;
801}
802
803bool RISCVDAGToDAGISel::tryUnsignedBitfieldInsertInZero(SDNode *Node,
804 const SDLoc &DL, MVT VT,
805 SDValue X, unsigned Msb,
806 unsigned Lsb) {
807 // Only supported with XAndesPerf at the moment.
808 if (!Subtarget->hasVendorXAndesPerf())
809 return false;
810
811 unsigned Opc = RISCV::NDS_BFOZ;
812
813 // If the Lsb is equal to the Msb, then the Lsb should be 0.
814 if (Lsb == Msb)
815 Lsb = 0;
816 SDNode *Ubi = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
817 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
818 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
819 ReplaceNode(F: Node, T: Ubi);
820 return true;
821}
822
823bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
824 // Target does not support indexed loads.
825 if (!Subtarget->hasVendorXTHeadMemIdx())
826 return false;
827
828 LoadSDNode *Ld = cast<LoadSDNode>(Val: Node);
829 ISD::MemIndexedMode AM = Ld->getAddressingMode();
830 if (AM == ISD::UNINDEXED)
831 return false;
832
833 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Ld->getOffset());
834 if (!C)
835 return false;
836
837 EVT LoadVT = Ld->getMemoryVT();
838 assert((AM == ISD::PRE_INC || AM == ISD::POST_INC) &&
839 "Unexpected addressing mode");
840 bool IsPre = AM == ISD::PRE_INC;
841 bool IsPost = AM == ISD::POST_INC;
842 int64_t Offset = C->getSExtValue();
843
844 // The constants that can be encoded in the THeadMemIdx instructions
845 // are of the form (sign_extend(imm5) << imm2).
846 unsigned Shift;
847 for (Shift = 0; Shift < 4; Shift++)
848 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
849 break;
850
851 // Constant cannot be encoded.
852 if (Shift == 4)
853 return false;
854
855 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
856 unsigned Opcode;
857 if (LoadVT == MVT::i8 && IsPre)
858 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
859 else if (LoadVT == MVT::i8 && IsPost)
860 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
861 else if (LoadVT == MVT::i16 && IsPre)
862 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
863 else if (LoadVT == MVT::i16 && IsPost)
864 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
865 else if (LoadVT == MVT::i32 && IsPre)
866 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
867 else if (LoadVT == MVT::i32 && IsPost)
868 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
869 else if (LoadVT == MVT::i64 && IsPre)
870 Opcode = RISCV::TH_LDIB;
871 else if (LoadVT == MVT::i64 && IsPost)
872 Opcode = RISCV::TH_LDIA;
873 else
874 return false;
875
876 EVT Ty = Ld->getOffset().getValueType();
877 SDValue Ops[] = {
878 Ld->getBasePtr(),
879 CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(Node), VT: Ty),
880 CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(Node), VT: Ty), Ld->getChain()};
881 SDNode *New = CurDAG->getMachineNode(Opcode, dl: SDLoc(Node), VT1: Ld->getValueType(ResNo: 0),
882 VT2: Ld->getValueType(ResNo: 1), VT3: MVT::Other, Ops);
883
884 MachineMemOperand *MemOp = cast<MemSDNode>(Val: Node)->getMemOperand();
885 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: New), NewMemRefs: {MemOp});
886
887 ReplaceNode(F: Node, T: New);
888
889 return true;
890}
891
892static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT,
893 SDValue Lo, SDValue Hi) {
894 SDValue Ops[] = {
895 CurDAG->getTargetConstant(Val: RISCV::GPRPairRegClassID, DL, VT: MVT::i32), Lo,
896 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_even, DL, VT: MVT::i32), Hi,
897 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_odd, DL, VT: MVT::i32)};
898
899 return SDValue(
900 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT, Ops), 0);
901}
902
903// Helper to extract Lo and Hi values from a GPR pair.
904static std::pair<SDValue, SDValue>
905extractGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, SDValue Pair) {
906 SDValue Lo =
907 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL, VT: MVT::i32, Operand: Pair);
908 SDValue Hi =
909 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_odd, DL, VT: MVT::i32, Operand: Pair);
910 return {Lo, Hi};
911}
912
913// Try to match WMACC pattern: ADDD where one operand pair comes from a
914// widening multiply (both results of UMUL_LOHI, SMUL_LOHI, or WMULSU).
915bool RISCVDAGToDAGISel::tryWideningMulAcc(SDNode *Node, const SDLoc &DL) {
916 assert(Node->getOpcode() == RISCVISD::ADDD && "Expected ADDD");
917
918 SDValue Op0Lo = Node->getOperand(Num: 0);
919 SDValue Op0Hi = Node->getOperand(Num: 1);
920 SDValue Op1Lo = Node->getOperand(Num: 2);
921 SDValue Op1Hi = Node->getOperand(Num: 3);
922
923 auto IsSupportedMulWithOneUse = [](SDValue Lo, SDValue Hi) {
924 unsigned Opc = Lo.getOpcode();
925 if (Opc != ISD::UMUL_LOHI && Opc != ISD::SMUL_LOHI &&
926 Opc != RISCVISD::WMULSU)
927 return false;
928 return Lo.getNode() == Hi.getNode() && Lo.getResNo() == 0 &&
929 Hi.getResNo() == 1 && Lo.hasOneUse() && Hi.hasOneUse();
930 };
931
932 SDNode *MulNode = nullptr;
933 SDValue AddLo, AddHi;
934
935 // Check if first operand pair is a supported multiply with single use.
936 if (IsSupportedMulWithOneUse(Op0Lo, Op0Hi)) {
937 MulNode = Op0Lo.getNode();
938 AddLo = Op1Lo;
939 AddHi = Op1Hi;
940 }
941 // ADDD is commutative. Check if second operand pair is a supported multiply
942 // with single use.
943 else if (IsSupportedMulWithOneUse(Op1Lo, Op1Hi)) {
944 MulNode = Op1Lo.getNode();
945 AddLo = Op0Lo;
946 AddHi = Op0Hi;
947 } else {
948 return false;
949 }
950
951 unsigned Opc;
952 switch (MulNode->getOpcode()) {
953 default:
954 llvm_unreachable("Unexpected multiply opcode");
955 case ISD::UMUL_LOHI:
956 Opc = RISCV::WMACCU;
957 break;
958 case ISD::SMUL_LOHI:
959 Opc = RISCV::WMACC;
960 break;
961 case RISCVISD::WMULSU:
962 Opc = RISCV::WMACCSU;
963 break;
964 }
965
966 SDValue Acc = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: AddLo, Hi: AddHi);
967
968 // WMACC instruction format: rd, rs1, rs2 (rd is accumulator).
969 SDValue M0 = MulNode->getOperand(Num: 0);
970 SDValue M1 = MulNode->getOperand(Num: 1);
971 MachineSDNode *New =
972 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Acc, Op2: M0, Op3: M1);
973
974 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
975 ReplaceUses(F: SDValue(Node, 0), T: Lo);
976 ReplaceUses(F: SDValue(Node, 1), T: Hi);
977 CurDAG->RemoveDeadNode(N: Node);
978 return true;
979}
980
981static Register getTileReg(uint64_t TileNum) {
982 assert(TileNum <= 15 && "Invalid tile number");
983 return RISCV::T0 + TileNum;
984}
985
986void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
987 if (!Subtarget->hasVInstructions())
988 return;
989
990 assert(Node->getOpcode() == ISD::INTRINSIC_VOID && "Unexpected opcode");
991
992 SDLoc DL(Node);
993 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
994
995 assert((IntNo == Intrinsic::riscv_sf_vc_x_se ||
996 IntNo == Intrinsic::riscv_sf_vc_i_se) &&
997 "Unexpected vsetvli intrinsic");
998
999 // imm, imm, imm, simm5/scalar, sew, log2lmul, vl
1000 unsigned Log2SEW = Log2_32(Value: Node->getConstantOperandVal(Num: 6));
1001 SDValue SEWOp =
1002 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: Subtarget->getXLenVT());
1003 SmallVector<SDValue, 8> Operands = {Node->getOperand(Num: 2), Node->getOperand(Num: 3),
1004 Node->getOperand(Num: 4), Node->getOperand(Num: 5),
1005 Node->getOperand(Num: 8), SEWOp,
1006 Node->getOperand(Num: 0)};
1007
1008 unsigned Opcode;
1009 auto *LMulSDNode = cast<ConstantSDNode>(Val: Node->getOperand(Num: 7));
1010 switch (LMulSDNode->getSExtValue()) {
1011 case 5:
1012 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
1013 : RISCV::PseudoSF_VC_I_SE_MF8;
1014 break;
1015 case 6:
1016 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
1017 : RISCV::PseudoSF_VC_I_SE_MF4;
1018 break;
1019 case 7:
1020 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
1021 : RISCV::PseudoSF_VC_I_SE_MF2;
1022 break;
1023 case 0:
1024 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
1025 : RISCV::PseudoSF_VC_I_SE_M1;
1026 break;
1027 case 1:
1028 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
1029 : RISCV::PseudoSF_VC_I_SE_M2;
1030 break;
1031 case 2:
1032 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
1033 : RISCV::PseudoSF_VC_I_SE_M4;
1034 break;
1035 case 3:
1036 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
1037 : RISCV::PseudoSF_VC_I_SE_M8;
1038 break;
1039 }
1040
1041 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
1042 Opcode, dl: DL, VT: Node->getSimpleValueType(ResNo: 0), Ops: Operands));
1043}
1044
1045static unsigned getSegInstNF(unsigned Intrinsic) {
1046#define INST_NF_CASE(NAME, NF) \
1047 case Intrinsic::riscv_##NAME##NF: \
1048 return NF;
1049#define INST_NF_CASE_MASK(NAME, NF) \
1050 case Intrinsic::riscv_##NAME##NF##_mask: \
1051 return NF;
1052#define INST_NF_CASE_FF(NAME, NF) \
1053 case Intrinsic::riscv_##NAME##NF##ff: \
1054 return NF;
1055#define INST_NF_CASE_FF_MASK(NAME, NF) \
1056 case Intrinsic::riscv_##NAME##NF##ff_mask: \
1057 return NF;
1058#define INST_ALL_NF_CASE_BASE(MACRO_NAME, NAME) \
1059 MACRO_NAME(NAME, 2) \
1060 MACRO_NAME(NAME, 3) \
1061 MACRO_NAME(NAME, 4) \
1062 MACRO_NAME(NAME, 5) \
1063 MACRO_NAME(NAME, 6) \
1064 MACRO_NAME(NAME, 7) \
1065 MACRO_NAME(NAME, 8)
1066#define INST_ALL_NF_CASE(NAME) \
1067 INST_ALL_NF_CASE_BASE(INST_NF_CASE, NAME) \
1068 INST_ALL_NF_CASE_BASE(INST_NF_CASE_MASK, NAME)
1069#define INST_ALL_NF_CASE_WITH_FF(NAME) \
1070 INST_ALL_NF_CASE(NAME) \
1071 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF, NAME) \
1072 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF_MASK, NAME)
1073 switch (Intrinsic) {
1074 default:
1075 llvm_unreachable("Unexpected segment load/store intrinsic");
1076 INST_ALL_NF_CASE_WITH_FF(vlseg)
1077 INST_ALL_NF_CASE(vlsseg)
1078 INST_ALL_NF_CASE(vloxseg)
1079 INST_ALL_NF_CASE(vluxseg)
1080 INST_ALL_NF_CASE(vsseg)
1081 INST_ALL_NF_CASE(vssseg)
1082 INST_ALL_NF_CASE(vsoxseg)
1083 INST_ALL_NF_CASE(vsuxseg)
1084 }
1085}
1086
1087static bool isApplicableToPLIOrPLUI(int Val) {
1088 // Check if the immediate is packed i8 or i10
1089 int16_t Bit31To16 = Val >> 16;
1090 int16_t Bit15To0 = Val;
1091 int8_t Bit15To8 = Bit15To0 >> 8;
1092 int8_t Bit7To0 = Val;
1093 if (Bit31To16 != Bit15To0)
1094 return false;
1095
1096 return isInt<10>(x: Bit15To0) || isShiftedInt<10, 6>(x: Bit15To0) ||
1097 Bit15To8 == Bit7To0;
1098}
1099
1100void RISCVDAGToDAGISel::Select(SDNode *Node) {
1101 // If we have a custom node, we have already selected.
1102 if (Node->isMachineOpcode()) {
1103 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
1104 Node->setNodeId(-1);
1105 return;
1106 }
1107
1108 // Instruction Selection not handled by the auto-generated tablegen selection
1109 // should be handled here.
1110 unsigned Opcode = Node->getOpcode();
1111 MVT XLenVT = Subtarget->getXLenVT();
1112 SDLoc DL(Node);
1113 MVT VT = Node->getSimpleValueType(ResNo: 0);
1114
1115 bool HasBitTest = Subtarget->hasBEXTILike();
1116
1117 switch (Opcode) {
1118 case ISD::Constant: {
1119 assert(VT == Subtarget->getXLenVT() && "Unexpected VT");
1120 auto *ConstNode = cast<ConstantSDNode>(Val: Node);
1121 if (ConstNode->isZero()) {
1122 SDValue New =
1123 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: RISCV::X0, VT);
1124 ReplaceNode(F: Node, T: New.getNode());
1125 return;
1126 }
1127 int64_t Imm = ConstNode->getSExtValue();
1128 // If only the lower 8 bits are used, try to convert this to a simm6 by
1129 // sign-extending bit 7. This is neutral without the C extension, and
1130 // allows C.LI to be used if C is present.
1131 if (!isInt<8>(x: Imm) && isUInt<8>(x: Imm) && isInt<6>(x: SignExtend64<8>(x: Imm)) &&
1132 hasAllBUsers(Node))
1133 Imm = SignExtend64<8>(x: Imm);
1134 // If the upper XLen-16 bits are not used, try to convert this to a simm12
1135 // by sign extending bit 15.
1136 else if (!isInt<16>(x: Imm) && isUInt<16>(x: Imm) &&
1137 isInt<12>(x: SignExtend64<16>(x: Imm)) && hasAllHUsers(Node))
1138 Imm = SignExtend64<16>(x: Imm);
1139
1140 // If the upper XLen-16 bits are not used, the lower 2 bytes are the same,
1141 // and we can't use li, convert to an xlen splat so we can use pli.b.
1142 if (Subtarget->hasStdExtP() && !isInt<12>(x: Imm) &&
1143 (Imm & 0xff) == ((Imm >> 8) & 0xff) && hasAllHUsers(Node)) {
1144 // Splat the lower 16 bits to XLen. Sign extend for RV32.
1145 uint64_t Splat = Imm & 0xffff;
1146 Splat = (Splat << 16) | Splat;
1147 if (VT == MVT::i64)
1148 Imm = Splat << 32 | Splat;
1149 else
1150 Imm = SignExtend64<32>(x: Splat);
1151 } else {
1152 // If the upper 32-bits are not used try to convert this into a simm32 by
1153 // sign extending bit 32.
1154 if (!isInt<32>(x: Imm) && isUInt<32>(x: Imm) && hasAllWUsers(Node))
1155 Imm = SignExtend64<32>(x: Imm);
1156
1157 if (VT == MVT::i64 && !isInt<12>(x: Imm) && !isShiftedInt<20, 12>(x: Imm) &&
1158 Subtarget->hasStdExtP() && isApplicableToPLIOrPLUI(Val: Imm) &&
1159 hasAllWUsers(Node)) {
1160 // If it's 4 packed 8-bit integers or 2 packed signed 16-bit integers,
1161 // we can simply copy lower 32 bits to higher 32 bits to make it able to
1162 // rematerialize to PLI_B or PLI_H
1163 Imm = ((uint64_t)Imm << 32) | (Imm & 0xFFFFFFFF);
1164 }
1165 }
1166
1167 ReplaceNode(F: Node, T: selectImm(CurDAG, DL, VT, Imm, Subtarget: *Subtarget).getNode());
1168 return;
1169 }
1170 case ISD::ConstantFP: {
1171 const APFloat &APF = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1172
1173 bool Is64Bit = Subtarget->is64Bit();
1174 bool HasZdinx = Subtarget->hasStdExtZdinx();
1175
1176 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
1177 SDValue Imm;
1178 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
1179 // create an integer immediate.
1180 if (APF.isPosZero() || NegZeroF64) {
1181 if (VT == MVT::f64 && HasZdinx && !Is64Bit)
1182 Imm = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::f64);
1183 else
1184 Imm = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
1185 } else {
1186 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
1187 Subtarget: *Subtarget);
1188 }
1189
1190 unsigned Opc;
1191 switch (VT.SimpleTy) {
1192 default:
1193 llvm_unreachable("Unexpected size");
1194 case MVT::bf16:
1195 assert(Subtarget->hasStdExtZfbfmin());
1196 Opc = RISCV::FMV_H_X;
1197 break;
1198 case MVT::f16:
1199 Opc = Subtarget->hasStdExtZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
1200 break;
1201 case MVT::f32:
1202 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
1203 break;
1204 case MVT::f64:
1205 // For RV32, we can't move from a GPR, we need to convert instead. This
1206 // should only happen for +0.0 and -0.0.
1207 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
1208 if (HasZdinx)
1209 Opc = RISCV::COPY;
1210 else
1211 Opc = Is64Bit ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
1212 break;
1213 }
1214
1215 SDNode *Res;
1216 if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
1217 Res =
1218 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_16, DL, VT, Operand: Imm).getNode();
1219 } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
1220 Res =
1221 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_32, DL, VT, Operand: Imm).getNode();
1222 } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
1223 Res = CurDAG->getMachineNode(
1224 Opcode: Opc, dl: DL, VT, Op1: Imm,
1225 Op2: CurDAG->getTargetConstant(Val: RISCVFPRndMode::RNE, DL, VT: XLenVT));
1226 else
1227 Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: Imm);
1228
1229 // For f64 -0.0, we need to insert a fneg.d idiom.
1230 if (NegZeroF64) {
1231 Opc = RISCV::FSGNJN_D;
1232 if (HasZdinx)
1233 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
1234 Res =
1235 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: SDValue(Res, 0), Op2: SDValue(Res, 0));
1236 }
1237
1238 ReplaceNode(F: Node, T: Res);
1239 return;
1240 }
1241 case RISCVISD::BuildGPRPair:
1242 case RISCVISD::BuildPairF64:
1243 case RISCVISD::BuildPairGPRVec: {
1244 if (Opcode == RISCVISD::BuildPairF64 && !Subtarget->hasStdExtZdinx())
1245 break;
1246
1247 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::BuildPairF64) &&
1248 "BuildPairF64 only handled here on rv32i_zdinx");
1249
1250 SDValue N =
1251 buildGPRPair(CurDAG, DL, VT, Lo: Node->getOperand(Num: 0), Hi: Node->getOperand(Num: 1));
1252 ReplaceNode(F: Node, T: N.getNode());
1253 return;
1254 }
1255 case RISCVISD::SplitGPRPair:
1256 case RISCVISD::SplitF64:
1257 case RISCVISD::SplitGPRVec: {
1258 if (Subtarget->hasStdExtZdinx() || Opcode != RISCVISD::SplitF64) {
1259 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::SplitF64) &&
1260 "SplitF64 only handled here on rv32i_zdinx");
1261
1262 if (!SDValue(Node, 0).use_empty()) {
1263 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL,
1264 VT: Node->getValueType(ResNo: 0),
1265 Operand: Node->getOperand(Num: 0));
1266 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1267 }
1268
1269 if (!SDValue(Node, 1).use_empty()) {
1270 SDValue Hi = CurDAG->getTargetExtractSubreg(
1271 SRIdx: RISCV::sub_gpr_odd, DL, VT: Node->getValueType(ResNo: 1), Operand: Node->getOperand(Num: 0));
1272 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1273 }
1274
1275 CurDAG->RemoveDeadNode(N: Node);
1276 return;
1277 }
1278
1279 if (!Subtarget->hasStdExtZfa())
1280 break;
1281 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
1282 "Unexpected subtarget");
1283
1284 // With Zfa, lower to fmv.x.w and fmvh.x.d.
1285 if (!SDValue(Node, 0).use_empty()) {
1286 SDNode *Lo = CurDAG->getMachineNode(Opcode: RISCV::FMV_X_W_FPR64, dl: DL, VT,
1287 Op1: Node->getOperand(Num: 0));
1288 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Lo, 0));
1289 }
1290 if (!SDValue(Node, 1).use_empty()) {
1291 SDNode *Hi = CurDAG->getMachineNode(Opcode: RISCV::FMVH_X_D, dl: DL, VT,
1292 Op1: Node->getOperand(Num: 0));
1293 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Hi, 0));
1294 }
1295
1296 CurDAG->RemoveDeadNode(N: Node);
1297 return;
1298 }
1299 case ISD::SHL: {
1300 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1301 if (!N1C)
1302 break;
1303 SDValue N0 = Node->getOperand(Num: 0);
1304 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
1305 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1306 break;
1307 unsigned ShAmt = N1C->getZExtValue();
1308 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1309
1310 if (isShiftedMask_64(Value: Mask)) {
1311 unsigned XLen = Subtarget->getXLen();
1312 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1313 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1314 if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
1315 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
1316 // where C2 has 32 leading zeros and C3 trailing zeros.
1317 SDNode *SRLIW = CurDAG->getMachineNode(
1318 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1319 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1320 SDNode *SLLI = CurDAG->getMachineNode(
1321 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1322 Op2: CurDAG->getTargetConstant(Val: TrailingZeros + ShAmt, DL, VT));
1323 ReplaceNode(F: Node, T: SLLI);
1324 return;
1325 }
1326 if (TrailingZeros == 0 && LeadingZeros > ShAmt &&
1327 XLen - LeadingZeros > 11 && LeadingZeros != 32) {
1328 // Optimize (shl (and X, C2), C) -> (srli (slli X, C4), C4-C)
1329 // where C2 has C4 leading zeros and no trailing zeros.
1330 // This is profitable if the "and" was to be lowered to
1331 // (srli (slli X, C4), C4) and not (andi X, C2).
1332 // For "LeadingZeros == 32":
1333 // - with Zba it's just (slli.uw X, C)
1334 // - without Zba a tablegen pattern applies the very same
1335 // transform as we would have done here
1336 SDNode *SLLI = CurDAG->getMachineNode(
1337 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1338 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1339 SDNode *SRLI = CurDAG->getMachineNode(
1340 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1341 Op2: CurDAG->getTargetConstant(Val: LeadingZeros - ShAmt, DL, VT));
1342 ReplaceNode(F: Node, T: SRLI);
1343 return;
1344 }
1345 }
1346 break;
1347 }
1348 case ISD::SRL: {
1349 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1350 if (!N1C)
1351 break;
1352 SDValue N0 = Node->getOperand(Num: 0);
1353 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1354 break;
1355 unsigned ShAmt = N1C->getZExtValue();
1356 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1357
1358 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1359 // 32 leading zeros and C3 trailing zeros.
1360 if (isShiftedMask_64(Value: Mask) && N0.hasOneUse()) {
1361 unsigned XLen = Subtarget->getXLen();
1362 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1363 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1364 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1365 SDNode *SRLIW = CurDAG->getMachineNode(
1366 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1367 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1368 SDNode *SLLI = CurDAG->getMachineNode(
1369 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1370 Op2: CurDAG->getTargetConstant(Val: TrailingZeros - ShAmt, DL, VT));
1371 ReplaceNode(F: Node, T: SLLI);
1372 return;
1373 }
1374 }
1375
1376 // Optimize (srl (and X, C2), C) ->
1377 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1378 // Where C2 is a mask with C3 trailing ones.
1379 // Taking into account that the C2 may have had lower bits unset by
1380 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1381 // This pattern occurs when type legalizing right shifts for types with
1382 // less than XLen bits.
1383 Mask |= maskTrailingOnes<uint64_t>(N: ShAmt);
1384 if (!isMask_64(Value: Mask))
1385 break;
1386 unsigned TrailingOnes = llvm::countr_one(Value: Mask);
1387 if (ShAmt >= TrailingOnes)
1388 break;
1389 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1390 if (TrailingOnes == 32) {
1391 SDNode *SRLI = CurDAG->getMachineNode(
1392 Opcode: Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, dl: DL, VT,
1393 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1394 ReplaceNode(F: Node, T: SRLI);
1395 return;
1396 }
1397
1398 // Only do the remaining transforms if the AND has one use.
1399 if (!N0.hasOneUse())
1400 break;
1401
1402 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1403 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1404 SDNode *BEXTI = CurDAG->getMachineNode(
1405 Opcode: Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, dl: DL, VT,
1406 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1407 ReplaceNode(F: Node, T: BEXTI);
1408 return;
1409 }
1410
1411 const unsigned Msb = TrailingOnes - 1;
1412 const unsigned Lsb = ShAmt;
1413 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0.getOperand(i: 0), Msb, Lsb))
1414 return;
1415
1416 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1417 SDNode *SLLI =
1418 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1419 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1420 SDNode *SRLI = CurDAG->getMachineNode(
1421 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1422 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1423 ReplaceNode(F: Node, T: SRLI);
1424 return;
1425 }
1426 case ISD::SRA: {
1427 if (trySignedBitfieldExtract(Node))
1428 return;
1429
1430 if (trySignedBitfieldInsertInSign(Node))
1431 return;
1432
1433 // Optimize (sra (sext_inreg X, i16), C) ->
1434 // (srai (slli X, (XLen-16), (XLen-16) + C)
1435 // And (sra (sext_inreg X, i8), C) ->
1436 // (srai (slli X, (XLen-8), (XLen-8) + C)
1437 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1438 // This transform matches the code we get without Zbb. The shifts are more
1439 // compressible, and this can help expose CSE opportunities in the sdiv by
1440 // constant optimization.
1441 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1442 if (!N1C)
1443 break;
1444 SDValue N0 = Node->getOperand(Num: 0);
1445 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1446 break;
1447 unsigned ShAmt = N1C->getZExtValue();
1448 unsigned ExtSize =
1449 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
1450 // ExtSize of 32 should use sraiw via tablegen pattern.
1451 if (ExtSize >= 32 || ShAmt >= ExtSize)
1452 break;
1453 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1454 SDNode *SLLI =
1455 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1456 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1457 SDNode *SRAI = CurDAG->getMachineNode(
1458 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1459 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1460 ReplaceNode(F: Node, T: SRAI);
1461 return;
1462 }
1463 case ISD::SIGN_EXTEND_INREG: {
1464 // Optimize (sext_inreg (srl X, C), i8/i16) ->
1465 // (srai (slli X, XLen-ExtSize-C), XLen-ExtSize)
1466 // This is a bitfield extract pattern where we're extracting a signed
1467 // 8-bit or 16-bit field from position C.
1468 SDValue N0 = Node->getOperand(Num: 0);
1469 if (N0.getOpcode() != ISD::SRL || !N0.hasOneUse())
1470 break;
1471
1472 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1473 if (!ShAmtC)
1474 break;
1475
1476 unsigned ExtSize =
1477 cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT().getSizeInBits();
1478 unsigned ShAmt = ShAmtC->getZExtValue();
1479 unsigned XLen = Subtarget->getXLen();
1480
1481 // Only handle types less than 32, and make sure the shift amount is valid.
1482 if (ExtSize >= 32 || ShAmt >= XLen - ExtSize)
1483 break;
1484
1485 unsigned LShAmt = XLen - ExtSize - ShAmt;
1486 SDNode *SLLI =
1487 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1488 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1489 SDNode *SRAI = CurDAG->getMachineNode(
1490 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1491 Op2: CurDAG->getTargetConstant(Val: XLen - ExtSize, DL, VT));
1492 ReplaceNode(F: Node, T: SRAI);
1493 return;
1494 }
1495 case ISD::OR: {
1496 if (tryShrinkShlLogicImm(Node))
1497 return;
1498
1499 break;
1500 }
1501 case ISD::XOR:
1502 if (tryShrinkShlLogicImm(Node))
1503 return;
1504
1505 break;
1506 case ISD::AND: {
1507 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1508 if (!N1C)
1509 break;
1510
1511 SDValue N0 = Node->getOperand(Num: 0);
1512
1513 bool LeftShift = N0.getOpcode() == ISD::SHL;
1514 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1515 auto *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1516 if (!C)
1517 break;
1518 unsigned C2 = C->getZExtValue();
1519 unsigned XLen = Subtarget->getXLen();
1520 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1521
1522 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1523 // shift pair might offer more compression opportunities.
1524 // TODO: We could check for C extension here, but we don't have many lit
1525 // tests with the C extension enabled so not checking gets better
1526 // coverage.
1527 // TODO: What if ANDI faster than shift?
1528 bool IsCANDI = isInt<6>(x: N1C->getSExtValue());
1529
1530 uint64_t C1 = N1C->getZExtValue();
1531
1532 // Clear irrelevant bits in the mask.
1533 if (LeftShift)
1534 C1 &= maskTrailingZeros<uint64_t>(N: C2);
1535 else
1536 C1 &= maskTrailingOnes<uint64_t>(N: XLen - C2);
1537
1538 // Some transforms should only be done if the shift has a single use or
1539 // the AND would become (srli (slli X, 32), 32)
1540 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1541
1542 SDValue X = N0.getOperand(i: 0);
1543
1544 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1545 // with c3 leading zeros.
1546 if (!LeftShift && isMask_64(Value: C1)) {
1547 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1548 if (C2 < Leading) {
1549 // If the number of leading zeros is C2+32 this can be SRLIW.
1550 if (C2 + 32 == Leading) {
1551 SDNode *SRLIW = CurDAG->getMachineNode(
1552 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X, Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1553 ReplaceNode(F: Node, T: SRLIW);
1554 return;
1555 }
1556
1557 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1558 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1559 //
1560 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1561 // legalized and goes through DAG combine.
1562 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1563 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1564 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32) {
1565 SDNode *SRAIW =
1566 CurDAG->getMachineNode(Opcode: RISCV::SRAIW, dl: DL, VT, Op1: X.getOperand(i: 0),
1567 Op2: CurDAG->getTargetConstant(Val: 31, DL, VT));
1568 SDNode *SRLIW = CurDAG->getMachineNode(
1569 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: SDValue(SRAIW, 0),
1570 Op2: CurDAG->getTargetConstant(Val: Leading - 32, DL, VT));
1571 ReplaceNode(F: Node, T: SRLIW);
1572 return;
1573 }
1574
1575 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1576 // available.
1577 // Transform (and (srl x, C2), C1)
1578 // -> (<bfextract> x, msb, lsb)
1579 //
1580 // Make sure to keep this below the SRLIW cases, as we always want to
1581 // prefer the more common instruction.
1582 const unsigned Msb = llvm::bit_width(Value: C1) + C2 - 1;
1583 const unsigned Lsb = C2;
1584 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1585 return;
1586
1587 // (srli (slli x, c3-c2), c3).
1588 // Skip if we could use (zext.w (sraiw X, C2)).
1589 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1590 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1591 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32;
1592 // Also Skip if we can use bexti or th.tst.
1593 Skip |= HasBitTest && Leading == XLen - 1;
1594 if (OneUseOrZExtW && !Skip) {
1595 SDNode *SLLI = CurDAG->getMachineNode(
1596 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1597 Op2: CurDAG->getTargetConstant(Val: Leading - C2, DL, VT));
1598 SDNode *SRLI = CurDAG->getMachineNode(
1599 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1600 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1601 ReplaceNode(F: Node, T: SRLI);
1602 return;
1603 }
1604 }
1605 }
1606
1607 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1608 // shifted by c2 bits with c3 leading zeros.
1609 if (LeftShift && isShiftedMask_64(Value: C1)) {
1610 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1611
1612 if (C2 + Leading < XLen &&
1613 C1 == (maskTrailingOnes<uint64_t>(N: XLen - (C2 + Leading)) << C2)) {
1614 // Use slli.uw when possible.
1615 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1616 SDNode *SLLI_UW =
1617 CurDAG->getMachineNode(Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: X,
1618 Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1619 ReplaceNode(F: Node, T: SLLI_UW);
1620 return;
1621 }
1622
1623 // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
1624 // available.
1625 // Transform (and (shl x, c2), c1)
1626 // -> (<bfinsert> x, msb, lsb)
1627 // e.g.
1628 // (and (shl x, 12), 0x00fff000)
1629 // If XLen = 32 and C2 = 12, then
1630 // Msb = 32 - 8 - 1 = 23 and Lsb = 12
1631 const unsigned Msb = XLen - Leading - 1;
1632 const unsigned Lsb = C2;
1633 if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
1634 return;
1635
1636 if (OneUseOrZExtW && !IsCANDI) {
1637 // (packh x0, X)
1638 if (Subtarget->hasStdExtZbkb() && C1 == 0xff00 && C2 == 8) {
1639 SDNode *PACKH = CurDAG->getMachineNode(
1640 Opcode: RISCV::PACKH, dl: DL, VT,
1641 Op1: CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT()), Op2: X);
1642 ReplaceNode(F: Node, T: PACKH);
1643 return;
1644 }
1645 // (srli (slli c2+c3), c3)
1646 SDNode *SLLI = CurDAG->getMachineNode(
1647 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1648 Op2: CurDAG->getTargetConstant(Val: C2 + Leading, DL, VT));
1649 SDNode *SRLI = CurDAG->getMachineNode(
1650 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1651 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1652 ReplaceNode(F: Node, T: SRLI);
1653 return;
1654 }
1655 }
1656 }
1657
1658 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1659 // shifted mask with c2 leading zeros and c3 trailing zeros.
1660 if (!LeftShift && isShiftedMask_64(Value: C1)) {
1661 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1662 unsigned Trailing = llvm::countr_zero(Val: C1);
1663 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1664 !IsCANDI) {
1665 unsigned SrliOpc = RISCV::SRLI;
1666 // If the input is zexti32 we should use SRLIW.
1667 if (X.getOpcode() == ISD::AND &&
1668 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1669 X.getConstantOperandVal(i: 1) == UINT64_C(0xFFFFFFFF)) {
1670 SrliOpc = RISCV::SRLIW;
1671 X = X.getOperand(i: 0);
1672 }
1673 SDNode *SRLI = CurDAG->getMachineNode(
1674 Opcode: SrliOpc, dl: DL, VT, Op1: X,
1675 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1676 SDNode *SLLI = CurDAG->getMachineNode(
1677 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1678 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1679 ReplaceNode(F: Node, T: SLLI);
1680 return;
1681 }
1682 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1683 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1684 OneUseOrZExtW && !IsCANDI) {
1685 SDNode *SRLIW = CurDAG->getMachineNode(
1686 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1687 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1688 SDNode *SLLI = CurDAG->getMachineNode(
1689 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1690 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1691 ReplaceNode(F: Node, T: SLLI);
1692 return;
1693 }
1694 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1695 if (Trailing > 0 && Leading + Trailing == 32 && C2 + Trailing < XLen &&
1696 OneUseOrZExtW && Subtarget->hasStdExtZba()) {
1697 SDNode *SRLI = CurDAG->getMachineNode(
1698 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1699 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1700 SDNode *SLLI_UW = CurDAG->getMachineNode(
1701 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1702 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1703 ReplaceNode(F: Node, T: SLLI_UW);
1704 return;
1705 }
1706 }
1707
1708 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1709 // shifted mask with no leading zeros and c3 trailing zeros.
1710 if (LeftShift && isShiftedMask_64(Value: C1)) {
1711 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1712 unsigned Trailing = llvm::countr_zero(Val: C1);
1713 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1714 SDNode *SRLI = CurDAG->getMachineNode(
1715 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1716 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1717 SDNode *SLLI = CurDAG->getMachineNode(
1718 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1719 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1720 ReplaceNode(F: Node, T: SLLI);
1721 return;
1722 }
1723 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1724 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1725 SDNode *SRLIW = CurDAG->getMachineNode(
1726 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1727 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1728 SDNode *SLLI = CurDAG->getMachineNode(
1729 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1730 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1731 ReplaceNode(F: Node, T: SLLI);
1732 return;
1733 }
1734
1735 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1736 if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
1737 Subtarget->hasStdExtZba()) {
1738 SDNode *SRLI = CurDAG->getMachineNode(
1739 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1740 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1741 SDNode *SLLI_UW = CurDAG->getMachineNode(
1742 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1743 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1744 ReplaceNode(F: Node, T: SLLI_UW);
1745 return;
1746 }
1747 }
1748 }
1749
1750 const uint64_t C1 = N1C->getZExtValue();
1751
1752 if (N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
1753 N0.hasOneUse()) {
1754 unsigned C2 = N0.getConstantOperandVal(i: 1);
1755 unsigned XLen = Subtarget->getXLen();
1756 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1757
1758 SDValue X = N0.getOperand(i: 0);
1759
1760 // Prefer SRAIW + ANDI when possible.
1761 bool Skip = C2 > 32 && isInt<12>(x: N1C->getSExtValue()) &&
1762 X.getOpcode() == ISD::SHL &&
1763 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1764 X.getConstantOperandVal(i: 1) == 32;
1765 // Turn (and (sra x, c2), c1) -> (srli (srai x, c2-c3), c3) if c1 is a
1766 // mask with c3 leading zeros and c2 is larger than c3.
1767 if (isMask_64(Value: C1) && !Skip) {
1768 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1769 if (C2 > Leading) {
1770 SDNode *SRAI = CurDAG->getMachineNode(
1771 Opcode: RISCV::SRAI, dl: DL, VT, Op1: X,
1772 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1773 SDNode *SRLI = CurDAG->getMachineNode(
1774 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1775 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1776 ReplaceNode(F: Node, T: SRLI);
1777 return;
1778 }
1779 }
1780
1781 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
1782 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
1783 // use (slli (srli (srai y, c2 - c3), c3 + c4), c4).
1784 if (isShiftedMask_64(Value: C1) && !Skip) {
1785 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1786 unsigned Trailing = llvm::countr_zero(Val: C1);
1787 if (C2 > Leading && Leading > 0 && Trailing > 0) {
1788 SDNode *SRAI = CurDAG->getMachineNode(
1789 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1790 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1791 SDNode *SRLI = CurDAG->getMachineNode(
1792 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1793 Op2: CurDAG->getTargetConstant(Val: Leading + Trailing, DL, VT));
1794 SDNode *SLLI = CurDAG->getMachineNode(
1795 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1796 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1797 ReplaceNode(F: Node, T: SLLI);
1798 return;
1799 }
1800 }
1801 }
1802
1803 // If C1 masks off the upper bits only (but can't be formed as an
1804 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1805 // available.
1806 // Transform (and x, C1)
1807 // -> (<bfextract> x, msb, lsb)
1808 if (isMask_64(Value: C1) && !isInt<12>(x: N1C->getSExtValue()) &&
1809 !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
1810 !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
1811 const unsigned Msb = llvm::bit_width(Value: C1) - 1;
1812 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0, Msb, Lsb: 0))
1813 return;
1814 }
1815
1816 if (tryShrinkShlLogicImm(Node))
1817 return;
1818
1819 break;
1820 }
1821 case ISD::MUL: {
1822 // Special case for calculating (mul (and X, C2), C1) where the full product
1823 // fits in XLen bits. We can shift X left by the number of leading zeros in
1824 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1825 // product has XLen trailing zeros, putting it in the output of MULHU. This
1826 // can avoid materializing a constant in a register for C2.
1827
1828 // RHS should be a constant.
1829 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1830 if (!N1C || !N1C->hasOneUse())
1831 break;
1832
1833 // LHS should be an AND with constant.
1834 SDValue N0 = Node->getOperand(Num: 0);
1835 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1836 break;
1837
1838 uint64_t C2 = N0.getConstantOperandVal(i: 1);
1839
1840 // Constant should be a mask.
1841 if (!isMask_64(Value: C2))
1842 break;
1843
1844 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1845 // multiple users or the constant is a simm12. This prevents inserting a
1846 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1847 // make it more costly to materialize. Otherwise, using a SLLI might allow
1848 // it to be compressed.
1849 bool IsANDIOrZExt =
1850 isInt<12>(x: C2) ||
1851 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1852 // With XTHeadBb, we can use TH.EXTU.
1853 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1854 if (IsANDIOrZExt && (isInt<12>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1855 break;
1856 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1857 // the constant is a simm32.
1858 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1859 // With XTHeadBb, we can use TH.EXTU.
1860 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1861 if (IsZExtW && (isInt<32>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1862 break;
1863
1864 // We need to shift left the AND input and C1 by a total of XLen bits.
1865
1866 // How far left do we need to shift the AND input?
1867 unsigned XLen = Subtarget->getXLen();
1868 unsigned LeadingZeros = XLen - llvm::bit_width(Value: C2);
1869
1870 // The constant gets shifted by the remaining amount unless that would
1871 // shift bits out.
1872 uint64_t C1 = N1C->getZExtValue();
1873 unsigned ConstantShift = XLen - LeadingZeros;
1874 if (ConstantShift > (XLen - llvm::bit_width(Value: C1)))
1875 break;
1876
1877 uint64_t ShiftedC1 = C1 << ConstantShift;
1878 // If this RV32, we need to sign extend the constant.
1879 if (XLen == 32)
1880 ShiftedC1 = SignExtend64<32>(x: ShiftedC1);
1881
1882 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1883 SDNode *Imm = selectImm(CurDAG, DL, VT, Imm: ShiftedC1, Subtarget: *Subtarget).getNode();
1884 SDNode *SLLI =
1885 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1886 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1887 SDNode *MULHU = CurDAG->getMachineNode(Opcode: RISCV::MULHU, dl: DL, VT,
1888 Op1: SDValue(SLLI, 0), Op2: SDValue(Imm, 0));
1889 ReplaceNode(F: Node, T: MULHU);
1890 return;
1891 }
1892 case ISD::SMUL_LOHI:
1893 case ISD::UMUL_LOHI:
1894 case RISCVISD::WMULSU:
1895 case RISCVISD::WADD:
1896 case RISCVISD::WSUB:
1897 case RISCVISD::WADDU:
1898 case RISCVISD::WSUBU: {
1899 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1900 "Unexpected opcode");
1901
1902 unsigned Opc;
1903 switch (Node->getOpcode()) {
1904 default:
1905 llvm_unreachable("Unexpected opcode");
1906 case ISD::SMUL_LOHI:
1907 Opc = RISCV::WMUL;
1908 break;
1909 case ISD::UMUL_LOHI:
1910 Opc = RISCV::WMULU;
1911 break;
1912 case RISCVISD::WMULSU:
1913 Opc = RISCV::WMULSU;
1914 break;
1915 case RISCVISD::WADD:
1916 Opc = RISCV::WADD;
1917 break;
1918 case RISCVISD::WSUB:
1919 Opc = RISCV::WSUB;
1920 break;
1921 case RISCVISD::WADDU:
1922 Opc = RISCV::WADDU;
1923 break;
1924 case RISCVISD::WSUBU:
1925 Opc = RISCV::WSUBU;
1926 break;
1927 }
1928
1929 SDNode *Result = CurDAG->getMachineNode(
1930 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Node->getOperand(Num: 0), Op2: Node->getOperand(Num: 1));
1931
1932 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(Result, 0));
1933 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1934 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1935 CurDAG->RemoveDeadNode(N: Node);
1936 return;
1937 }
1938 case RISCVISD::WSLL:
1939 case RISCVISD::WSLA: {
1940 // Custom select WSLL/WSLA for RV32P.
1941 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1942 "Unexpected opcode");
1943
1944 bool IsSigned = Node->getOpcode() == RISCVISD::WSLA;
1945
1946 SDValue ShAmt = Node->getOperand(Num: 1);
1947
1948 unsigned Opc;
1949
1950 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShAmt);
1951 if (ShAmtC && ShAmtC->getZExtValue() < 64) {
1952 Opc = IsSigned ? RISCV::WSLAI : RISCV::WSLLI;
1953 ShAmt = CurDAG->getTargetConstant(Val: ShAmtC->getZExtValue(), DL, VT: XLenVT);
1954 } else {
1955 Opc = IsSigned ? RISCV::WSLA : RISCV::WSLL;
1956 }
1957
1958 SDNode *WShift = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped,
1959 Op1: Node->getOperand(Num: 0), Op2: ShAmt);
1960
1961 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(WShift, 0));
1962 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1963 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1964 CurDAG->RemoveDeadNode(N: Node);
1965 return;
1966 }
1967 case ISD::LOAD: {
1968 if (tryIndexedLoad(Node))
1969 return;
1970
1971 if (Subtarget->hasVendorXCVmem() && !Subtarget->is64Bit()) {
1972 // We match post-incrementing load here
1973 LoadSDNode *Load = cast<LoadSDNode>(Val: Node);
1974 if (Load->getAddressingMode() != ISD::POST_INC)
1975 break;
1976
1977 SDValue Chain = Node->getOperand(Num: 0);
1978 SDValue Base = Node->getOperand(Num: 1);
1979 SDValue Offset = Node->getOperand(Num: 2);
1980
1981 bool Simm12 = false;
1982 bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
1983
1984 if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Val&: Offset)) {
1985 int ConstantVal = ConstantOffset->getSExtValue();
1986 Simm12 = isInt<12>(x: ConstantVal);
1987 if (Simm12)
1988 Offset = CurDAG->getSignedTargetConstant(Val: ConstantVal, DL: SDLoc(Offset),
1989 VT: Offset.getValueType());
1990 }
1991
1992 unsigned Opcode = 0;
1993 switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
1994 case MVT::i8:
1995 if (Simm12 && SignExtend)
1996 Opcode = RISCV::CV_LB_ri_inc;
1997 else if (Simm12 && !SignExtend)
1998 Opcode = RISCV::CV_LBU_ri_inc;
1999 else if (!Simm12 && SignExtend)
2000 Opcode = RISCV::CV_LB_rr_inc;
2001 else
2002 Opcode = RISCV::CV_LBU_rr_inc;
2003 break;
2004 case MVT::i16:
2005 if (Simm12 && SignExtend)
2006 Opcode = RISCV::CV_LH_ri_inc;
2007 else if (Simm12 && !SignExtend)
2008 Opcode = RISCV::CV_LHU_ri_inc;
2009 else if (!Simm12 && SignExtend)
2010 Opcode = RISCV::CV_LH_rr_inc;
2011 else
2012 Opcode = RISCV::CV_LHU_rr_inc;
2013 break;
2014 case MVT::i32:
2015 if (Simm12)
2016 Opcode = RISCV::CV_LW_ri_inc;
2017 else
2018 Opcode = RISCV::CV_LW_rr_inc;
2019 break;
2020 default:
2021 break;
2022 }
2023 if (!Opcode)
2024 break;
2025
2026 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode, dl: DL, VT1: XLenVT, VT2: XLenVT,
2027 VT3: Chain.getSimpleValueType(), Op1: Base,
2028 Op2: Offset, Op3: Chain));
2029 return;
2030 }
2031 break;
2032 }
2033 case RISCVISD::LD_RV32: {
2034 assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd");
2035
2036 SDValue Base, Offset;
2037 SDValue Chain = Node->getOperand(Num: 0);
2038 SDValue Addr = Node->getOperand(Num: 1);
2039 SelectAddrRegImm(Addr, Base, Offset);
2040
2041 SDValue Ops[] = {Base, Offset, Chain};
2042 MachineSDNode *New = CurDAG->getMachineNode(
2043 Opcode: RISCV::LD_RV32, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2044 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2045 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2046 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2047 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2048 ReplaceUses(F: SDValue(Node, 2), T: SDValue(New, 1));
2049 CurDAG->RemoveDeadNode(N: Node);
2050 return;
2051 }
2052 case RISCVISD::SD_RV32: {
2053 SDValue Base, Offset;
2054 SDValue Chain = Node->getOperand(Num: 0);
2055 SDValue Addr = Node->getOperand(Num: 3);
2056 SelectAddrRegImm(Addr, Base, Offset);
2057
2058 SDValue Lo = Node->getOperand(Num: 1);
2059 SDValue Hi = Node->getOperand(Num: 2);
2060
2061 SDValue RegPair;
2062 // Peephole to use X0_Pair for storing zero.
2063 if (isNullConstant(V: Lo) && isNullConstant(V: Hi)) {
2064 RegPair = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2065 } else {
2066 RegPair = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo, Hi);
2067 }
2068
2069 MachineSDNode *New = CurDAG->getMachineNode(Opcode: RISCV::SD_RV32, dl: DL, VT: MVT::Other,
2070 Ops: {RegPair, Base, Offset, Chain});
2071 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2072 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0));
2073 CurDAG->RemoveDeadNode(N: Node);
2074 return;
2075 }
2076 case RISCVISD::MQWACC:
2077 case RISCVISD::MQRWACC: {
2078 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2079 "Unexpected opcode");
2080
2081 SDValue Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Node->getOperand(Num: 0),
2082 Hi: Node->getOperand(Num: 1));
2083 unsigned Opc = Opcode == RISCVISD::MQRWACC ? RISCV::MQRWACC : RISCV::MQWACC;
2084 MachineSDNode *New = CurDAG->getMachineNode(
2085 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Node->getOperand(Num: 2), Op3: Node->getOperand(Num: 3));
2086 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2087 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2088 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2089 CurDAG->RemoveDeadNode(N: Node);
2090 return;
2091 }
2092 case RISCVISD::ADDD:
2093 // Try to match WMACC pattern: ADDD where one operand pair comes from a
2094 // widening multiply.
2095 if (tryWideningMulAcc(Node, DL))
2096 return;
2097
2098 // Fall through to regular ADDD selection.
2099 [[fallthrough]];
2100 case RISCVISD::SUBD:
2101 case RISCVISD::WADDAU:
2102 case RISCVISD::WSUBAU:
2103 case RISCVISD::WADDA:
2104 case RISCVISD::WSUBA: {
2105 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2106 "Unexpected opcode");
2107
2108 SDValue Op0Lo = Node->getOperand(Num: 0);
2109 SDValue Op0Hi = Node->getOperand(Num: 1);
2110
2111 SDValue Op0;
2112 if (isNullConstant(V: Op0Lo) && isNullConstant(V: Op0Hi)) {
2113 Op0 = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2114 } else {
2115 Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op0Lo, Hi: Op0Hi);
2116 }
2117
2118 SDValue Op1Lo = Node->getOperand(Num: 2);
2119 SDValue Op1Hi = Node->getOperand(Num: 3);
2120
2121 MachineSDNode *New;
2122 if (Opcode == RISCVISD::WADDAU || Opcode == RISCVISD::WSUBAU ||
2123 Opcode == RISCVISD::WADDA || Opcode == RISCVISD::WSUBA) {
2124 // Widening accumulate: Op0 is the accumulator (GPRPair), Op1Lo and Op1Hi
2125 // are the two 32-bit values.
2126 unsigned Opc;
2127 switch (Opcode) {
2128 default:
2129 llvm_unreachable("Unexpected opcode");
2130 case RISCVISD::WADDAU:
2131 Opc = RISCV::WADDAU;
2132 break;
2133 case RISCVISD::WSUBAU:
2134 Opc = RISCV::WSUBAU;
2135 break;
2136 case RISCVISD::WADDA:
2137 Opc = RISCV::WADDA;
2138 break;
2139 case RISCVISD::WSUBA:
2140 Opc = RISCV::WSUBA;
2141 break;
2142 }
2143 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1Lo, Op3: Op1Hi);
2144 } else {
2145 SDValue Op1 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op1Lo, Hi: Op1Hi);
2146
2147 unsigned Opc;
2148 switch (Opcode) {
2149 default:
2150 llvm_unreachable("Unexpected opcode");
2151 case RISCVISD::ADDD:
2152 Opc = RISCV::ADDD;
2153 break;
2154 case RISCVISD::SUBD:
2155 Opc = RISCV::SUBD;
2156 break;
2157 }
2158 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1);
2159 }
2160
2161 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2162 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2163 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2164 CurDAG->RemoveDeadNode(N: Node);
2165 return;
2166 }
2167 case ISD::INTRINSIC_WO_CHAIN: {
2168 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
2169 switch (IntNo) {
2170 // By default we do not custom select any intrinsic.
2171 default:
2172 break;
2173 case Intrinsic::riscv_vmsgeu:
2174 case Intrinsic::riscv_vmsge: {
2175 SDValue Src1 = Node->getOperand(Num: 1);
2176 SDValue Src2 = Node->getOperand(Num: 2);
2177 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
2178 bool IsCmpConstant = false;
2179 bool IsCmpMinimum = false;
2180 // Only custom select scalar second operand.
2181 if (Src2.getValueType() != XLenVT)
2182 break;
2183 // Small constants are handled with patterns.
2184 int64_t CVal = 0;
2185 MVT Src1VT = Src1.getSimpleValueType();
2186 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2187 IsCmpConstant = true;
2188 CVal = C->getSExtValue();
2189 if (CVal >= -15 && CVal <= 16) {
2190 if (!IsUnsigned || CVal != 0)
2191 break;
2192 IsCmpMinimum = true;
2193 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2194 numBits: Src1VT.getScalarSizeInBits())
2195 .getSExtValue()) {
2196 IsCmpMinimum = true;
2197 }
2198 }
2199 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode, VMSGTOpcode;
2200 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2201 default:
2202 llvm_unreachable("Unexpected LMUL!");
2203#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2204 case RISCVVType::lmulenum: \
2205 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2206 : RISCV::PseudoVMSLT_VX_##suffix; \
2207 VMSGTOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix \
2208 : RISCV::PseudoVMSGT_VX_##suffix; \
2209 break;
2210 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2211 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2212 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2213 CASE_VMSLT_OPCODES(LMUL_1, M1)
2214 CASE_VMSLT_OPCODES(LMUL_2, M2)
2215 CASE_VMSLT_OPCODES(LMUL_4, M4)
2216 CASE_VMSLT_OPCODES(LMUL_8, M8)
2217#undef CASE_VMSLT_OPCODES
2218 }
2219 // Mask operations use the LMUL from the mask type.
2220 switch (RISCVTargetLowering::getLMUL(VT)) {
2221 default:
2222 llvm_unreachable("Unexpected LMUL!");
2223#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix) \
2224 case RISCVVType::lmulenum: \
2225 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
2226 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix; \
2227 break;
2228 CASE_VMNAND_VMSET_OPCODES(LMUL_F8, B64)
2229 CASE_VMNAND_VMSET_OPCODES(LMUL_F4, B32)
2230 CASE_VMNAND_VMSET_OPCODES(LMUL_F2, B16)
2231 CASE_VMNAND_VMSET_OPCODES(LMUL_1, B8)
2232 CASE_VMNAND_VMSET_OPCODES(LMUL_2, B4)
2233 CASE_VMNAND_VMSET_OPCODES(LMUL_4, B2)
2234 CASE_VMNAND_VMSET_OPCODES(LMUL_8, B1)
2235#undef CASE_VMNAND_VMSET_OPCODES
2236 }
2237 SDValue SEW = CurDAG->getTargetConstant(
2238 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2239 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2240 SDValue VL;
2241 selectVLOp(N: Node->getOperand(Num: 3), VL);
2242
2243 // If vmsge(u) with minimum value, expand it to vmset.
2244 if (IsCmpMinimum) {
2245 ReplaceNode(F: Node,
2246 T: CurDAG->getMachineNode(Opcode: VMSetOpcode, dl: DL, VT, Op1: VL, Op2: MaskSEW));
2247 return;
2248 }
2249
2250 if (IsCmpConstant) {
2251 SDValue Imm =
2252 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2253
2254 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMSGTOpcode, dl: DL, VT,
2255 Ops: {Src1, Imm, VL, SEW}));
2256 return;
2257 }
2258
2259 // Expand to
2260 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
2261 SDValue Cmp = SDValue(
2262 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2263 0);
2264 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMNANDOpcode, dl: DL, VT,
2265 Ops: {Cmp, Cmp, VL, MaskSEW}));
2266 return;
2267 }
2268 case Intrinsic::riscv_vmsgeu_mask:
2269 case Intrinsic::riscv_vmsge_mask: {
2270 SDValue Src1 = Node->getOperand(Num: 2);
2271 SDValue Src2 = Node->getOperand(Num: 3);
2272 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
2273 bool IsCmpConstant = false;
2274 bool IsCmpMinimum = false;
2275 // Only custom select scalar second operand.
2276 if (Src2.getValueType() != XLenVT)
2277 break;
2278 // Small constants are handled with patterns.
2279 MVT Src1VT = Src1.getSimpleValueType();
2280 int64_t CVal = 0;
2281 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2282 IsCmpConstant = true;
2283 CVal = C->getSExtValue();
2284 if (CVal >= -15 && CVal <= 16) {
2285 if (!IsUnsigned || CVal != 0)
2286 break;
2287 IsCmpMinimum = true;
2288 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2289 numBits: Src1VT.getScalarSizeInBits())
2290 .getSExtValue()) {
2291 IsCmpMinimum = true;
2292 }
2293 }
2294 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
2295 VMOROpcode, VMSGTMaskOpcode;
2296 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2297 default:
2298 llvm_unreachable("Unexpected LMUL!");
2299#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2300 case RISCVVType::lmulenum: \
2301 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2302 : RISCV::PseudoVMSLT_VX_##suffix; \
2303 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
2304 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
2305 VMSGTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix##_MASK \
2306 : RISCV::PseudoVMSGT_VX_##suffix##_MASK; \
2307 break;
2308 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2309 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2310 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2311 CASE_VMSLT_OPCODES(LMUL_1, M1)
2312 CASE_VMSLT_OPCODES(LMUL_2, M2)
2313 CASE_VMSLT_OPCODES(LMUL_4, M4)
2314 CASE_VMSLT_OPCODES(LMUL_8, M8)
2315#undef CASE_VMSLT_OPCODES
2316 }
2317 // Mask operations use the LMUL from the mask type.
2318 switch (RISCVTargetLowering::getLMUL(VT)) {
2319 default:
2320 llvm_unreachable("Unexpected LMUL!");
2321#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
2322 case RISCVVType::lmulenum: \
2323 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
2324 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
2325 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
2326 break;
2327 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, B64)
2328 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, B32)
2329 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, B16)
2330 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, B8)
2331 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, B4)
2332 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, B2)
2333 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, B1)
2334#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
2335 }
2336 SDValue SEW = CurDAG->getTargetConstant(
2337 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2338 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2339 SDValue VL;
2340 selectVLOp(N: Node->getOperand(Num: 5), VL);
2341 SDValue MaskedOff = Node->getOperand(Num: 1);
2342 SDValue Mask = Node->getOperand(Num: 4);
2343
2344 // If vmsge(u) with minimum value, expand it to vmor mask, maskedoff.
2345 if (IsCmpMinimum) {
2346 // We don't need vmor if the MaskedOff and the Mask are the same
2347 // value.
2348 if (Mask == MaskedOff) {
2349 ReplaceUses(F: Node, T: Mask.getNode());
2350 return;
2351 }
2352 ReplaceNode(F: Node,
2353 T: CurDAG->getMachineNode(Opcode: VMOROpcode, dl: DL, VT,
2354 Ops: {Mask, MaskedOff, VL, MaskSEW}));
2355 return;
2356 }
2357
2358 // If the MaskedOff value and the Mask are the same value use
2359 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
2360 // This avoids needing to copy v0 to vd before starting the next sequence.
2361 if (Mask == MaskedOff) {
2362 SDValue Cmp = SDValue(
2363 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2364 0);
2365 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMANDNOpcode, dl: DL, VT,
2366 Ops: {Mask, Cmp, VL, MaskSEW}));
2367 return;
2368 }
2369
2370 SDValue PolicyOp =
2371 CurDAG->getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
2372
2373 if (IsCmpConstant) {
2374 SDValue Imm =
2375 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2376
2377 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
2378 Opcode: VMSGTMaskOpcode, dl: DL, VT,
2379 Ops: {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp}));
2380 return;
2381 }
2382
2383 // Otherwise use
2384 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
2385 // The result is mask undisturbed.
2386 // We use the same instructions to emulate mask agnostic behavior, because
2387 // the agnostic result can be either undisturbed or all 1.
2388 SDValue Cmp = SDValue(CurDAG->getMachineNode(Opcode: VMSLTMaskOpcode, dl: DL, VT,
2389 Ops: {MaskedOff, Src1, Src2, Mask,
2390 VL, SEW, PolicyOp}),
2391 0);
2392 // vmxor.mm vd, vd, v0 is used to update active value.
2393 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMXOROpcode, dl: DL, VT,
2394 Ops: {Cmp, Mask, VL, MaskSEW}));
2395 return;
2396 }
2397 case Intrinsic::riscv_vsetvli:
2398 case Intrinsic::riscv_vsetvlimax:
2399 return selectVSETVLI(Node);
2400 case Intrinsic::riscv_sf_vsettnt:
2401 case Intrinsic::riscv_sf_vsettm:
2402 case Intrinsic::riscv_sf_vsettk:
2403 return selectXSfmmVSET(Node);
2404 }
2405 break;
2406 }
2407 case ISD::INTRINSIC_W_CHAIN: {
2408 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2409 switch (IntNo) {
2410 // By default we do not custom select any intrinsic.
2411 default:
2412 break;
2413 case Intrinsic::riscv_vlseg2:
2414 case Intrinsic::riscv_vlseg3:
2415 case Intrinsic::riscv_vlseg4:
2416 case Intrinsic::riscv_vlseg5:
2417 case Intrinsic::riscv_vlseg6:
2418 case Intrinsic::riscv_vlseg7:
2419 case Intrinsic::riscv_vlseg8: {
2420 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2421 /*IsStrided*/ false);
2422 return;
2423 }
2424 case Intrinsic::riscv_vlseg2_mask:
2425 case Intrinsic::riscv_vlseg3_mask:
2426 case Intrinsic::riscv_vlseg4_mask:
2427 case Intrinsic::riscv_vlseg5_mask:
2428 case Intrinsic::riscv_vlseg6_mask:
2429 case Intrinsic::riscv_vlseg7_mask:
2430 case Intrinsic::riscv_vlseg8_mask: {
2431 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2432 /*IsStrided*/ false);
2433 return;
2434 }
2435 case Intrinsic::riscv_vlsseg2:
2436 case Intrinsic::riscv_vlsseg3:
2437 case Intrinsic::riscv_vlsseg4:
2438 case Intrinsic::riscv_vlsseg5:
2439 case Intrinsic::riscv_vlsseg6:
2440 case Intrinsic::riscv_vlsseg7:
2441 case Intrinsic::riscv_vlsseg8: {
2442 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2443 /*IsStrided*/ true);
2444 return;
2445 }
2446 case Intrinsic::riscv_vlsseg2_mask:
2447 case Intrinsic::riscv_vlsseg3_mask:
2448 case Intrinsic::riscv_vlsseg4_mask:
2449 case Intrinsic::riscv_vlsseg5_mask:
2450 case Intrinsic::riscv_vlsseg6_mask:
2451 case Intrinsic::riscv_vlsseg7_mask:
2452 case Intrinsic::riscv_vlsseg8_mask: {
2453 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2454 /*IsStrided*/ true);
2455 return;
2456 }
2457 case Intrinsic::riscv_vloxseg2:
2458 case Intrinsic::riscv_vloxseg3:
2459 case Intrinsic::riscv_vloxseg4:
2460 case Intrinsic::riscv_vloxseg5:
2461 case Intrinsic::riscv_vloxseg6:
2462 case Intrinsic::riscv_vloxseg7:
2463 case Intrinsic::riscv_vloxseg8:
2464 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2465 /*IsOrdered*/ true);
2466 return;
2467 case Intrinsic::riscv_vluxseg2:
2468 case Intrinsic::riscv_vluxseg3:
2469 case Intrinsic::riscv_vluxseg4:
2470 case Intrinsic::riscv_vluxseg5:
2471 case Intrinsic::riscv_vluxseg6:
2472 case Intrinsic::riscv_vluxseg7:
2473 case Intrinsic::riscv_vluxseg8:
2474 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2475 /*IsOrdered*/ false);
2476 return;
2477 case Intrinsic::riscv_vloxseg2_mask:
2478 case Intrinsic::riscv_vloxseg3_mask:
2479 case Intrinsic::riscv_vloxseg4_mask:
2480 case Intrinsic::riscv_vloxseg5_mask:
2481 case Intrinsic::riscv_vloxseg6_mask:
2482 case Intrinsic::riscv_vloxseg7_mask:
2483 case Intrinsic::riscv_vloxseg8_mask:
2484 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2485 /*IsOrdered*/ true);
2486 return;
2487 case Intrinsic::riscv_vluxseg2_mask:
2488 case Intrinsic::riscv_vluxseg3_mask:
2489 case Intrinsic::riscv_vluxseg4_mask:
2490 case Intrinsic::riscv_vluxseg5_mask:
2491 case Intrinsic::riscv_vluxseg6_mask:
2492 case Intrinsic::riscv_vluxseg7_mask:
2493 case Intrinsic::riscv_vluxseg8_mask:
2494 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2495 /*IsOrdered*/ false);
2496 return;
2497 case Intrinsic::riscv_vlseg8ff:
2498 case Intrinsic::riscv_vlseg7ff:
2499 case Intrinsic::riscv_vlseg6ff:
2500 case Intrinsic::riscv_vlseg5ff:
2501 case Intrinsic::riscv_vlseg4ff:
2502 case Intrinsic::riscv_vlseg3ff:
2503 case Intrinsic::riscv_vlseg2ff: {
2504 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false);
2505 return;
2506 }
2507 case Intrinsic::riscv_vlseg8ff_mask:
2508 case Intrinsic::riscv_vlseg7ff_mask:
2509 case Intrinsic::riscv_vlseg6ff_mask:
2510 case Intrinsic::riscv_vlseg5ff_mask:
2511 case Intrinsic::riscv_vlseg4ff_mask:
2512 case Intrinsic::riscv_vlseg3ff_mask:
2513 case Intrinsic::riscv_vlseg2ff_mask: {
2514 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true);
2515 return;
2516 }
2517 case Intrinsic::riscv_vloxei:
2518 case Intrinsic::riscv_vloxei_mask:
2519 case Intrinsic::riscv_vluxei:
2520 case Intrinsic::riscv_vluxei_mask: {
2521 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
2522 IntNo == Intrinsic::riscv_vluxei_mask;
2523 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
2524 IntNo == Intrinsic::riscv_vloxei_mask;
2525
2526 MVT VT = Node->getSimpleValueType(ResNo: 0);
2527 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2528
2529 unsigned CurOp = 2;
2530 SmallVector<SDValue, 8> Operands;
2531 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2532
2533 MVT IndexVT;
2534 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2535 /*IsStridedOrIndexed*/ true, Operands,
2536 /*IsLoad=*/true, IndexVT: &IndexVT);
2537
2538 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2539 "Element count mismatch");
2540
2541 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2542 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2543 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2544 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2545 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2546 "index values when XLEN=32");
2547 }
2548 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
2549 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
2550 IndexLMUL: static_cast<unsigned>(IndexLMUL));
2551 MachineSDNode *Load =
2552 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2553
2554 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2555
2556 ReplaceNode(F: Node, T: Load);
2557 return;
2558 }
2559 case Intrinsic::riscv_vlm:
2560 case Intrinsic::riscv_vle:
2561 case Intrinsic::riscv_vle_mask:
2562 case Intrinsic::riscv_vlse:
2563 case Intrinsic::riscv_vlse_mask: {
2564 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
2565 IntNo == Intrinsic::riscv_vlse_mask;
2566 bool IsStrided =
2567 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
2568
2569 MVT VT = Node->getSimpleValueType(ResNo: 0);
2570 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2571
2572 // The riscv_vlm intrinsic are always tail agnostic and no passthru
2573 // operand at the IR level. In pseudos, they have both policy and
2574 // passthru operand. The passthru operand is needed to track the
2575 // "tail undefined" state, and the policy is there just for
2576 // for consistency - it will always be "don't care" for the
2577 // unmasked form.
2578 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
2579 unsigned CurOp = 2;
2580 SmallVector<SDValue, 8> Operands;
2581 if (HasPassthruOperand)
2582 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2583 else {
2584 // We eagerly lower to implicit_def (instead of undef), as we
2585 // otherwise fail to select nodes such as: nxv1i1 = undef
2586 SDNode *Passthru =
2587 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
2588 Operands.push_back(Elt: SDValue(Passthru, 0));
2589 }
2590 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2591 Operands, /*IsLoad=*/true);
2592
2593 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2594 const RISCV::VLEPseudo *P =
2595 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
2596 LMUL: static_cast<unsigned>(LMUL));
2597 MachineSDNode *Load =
2598 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2599
2600 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2601
2602 ReplaceNode(F: Node, T: Load);
2603 return;
2604 }
2605 case Intrinsic::riscv_vleff:
2606 case Intrinsic::riscv_vleff_mask: {
2607 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
2608
2609 MVT VT = Node->getSimpleValueType(ResNo: 0);
2610 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2611
2612 unsigned CurOp = 2;
2613 SmallVector<SDValue, 7> Operands;
2614 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2615 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2616 /*IsStridedOrIndexed*/ false, Operands,
2617 /*IsLoad=*/true);
2618
2619 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2620 const RISCV::VLEPseudo *P =
2621 RISCV::getVLEPseudo(Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
2622 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2623 MachineSDNode *Load = CurDAG->getMachineNode(
2624 Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2625 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2626
2627 ReplaceNode(F: Node, T: Load);
2628 return;
2629 }
2630 case Intrinsic::riscv_nds_vln:
2631 case Intrinsic::riscv_nds_vln_mask:
2632 case Intrinsic::riscv_nds_vlnu:
2633 case Intrinsic::riscv_nds_vlnu_mask: {
2634 bool IsMasked = IntNo == Intrinsic::riscv_nds_vln_mask ||
2635 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2636 bool IsUnsigned = IntNo == Intrinsic::riscv_nds_vlnu ||
2637 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2638
2639 MVT VT = Node->getSimpleValueType(ResNo: 0);
2640 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2641 unsigned CurOp = 2;
2642 SmallVector<SDValue, 8> Operands;
2643
2644 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2645 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2646 /*IsStridedOrIndexed=*/false, Operands,
2647 /*IsLoad=*/true);
2648
2649 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2650 const RISCV::NDSVLNPseudo *P = RISCV::getNDSVLNPseudo(
2651 Masked: IsMasked, Unsigned: IsUnsigned, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2652 MachineSDNode *Load =
2653 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2654
2655 if (auto *MemOp = dyn_cast<MemSDNode>(Val: Node))
2656 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {MemOp->getMemOperand()});
2657
2658 ReplaceNode(F: Node, T: Load);
2659 return;
2660 }
2661 }
2662 break;
2663 }
2664 case ISD::INTRINSIC_VOID: {
2665 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2666 switch (IntNo) {
2667 case Intrinsic::riscv_vsseg2:
2668 case Intrinsic::riscv_vsseg3:
2669 case Intrinsic::riscv_vsseg4:
2670 case Intrinsic::riscv_vsseg5:
2671 case Intrinsic::riscv_vsseg6:
2672 case Intrinsic::riscv_vsseg7:
2673 case Intrinsic::riscv_vsseg8: {
2674 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2675 /*IsStrided*/ false);
2676 return;
2677 }
2678 case Intrinsic::riscv_vsseg2_mask:
2679 case Intrinsic::riscv_vsseg3_mask:
2680 case Intrinsic::riscv_vsseg4_mask:
2681 case Intrinsic::riscv_vsseg5_mask:
2682 case Intrinsic::riscv_vsseg6_mask:
2683 case Intrinsic::riscv_vsseg7_mask:
2684 case Intrinsic::riscv_vsseg8_mask: {
2685 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2686 /*IsStrided*/ false);
2687 return;
2688 }
2689 case Intrinsic::riscv_vssseg2:
2690 case Intrinsic::riscv_vssseg3:
2691 case Intrinsic::riscv_vssseg4:
2692 case Intrinsic::riscv_vssseg5:
2693 case Intrinsic::riscv_vssseg6:
2694 case Intrinsic::riscv_vssseg7:
2695 case Intrinsic::riscv_vssseg8: {
2696 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2697 /*IsStrided*/ true);
2698 return;
2699 }
2700 case Intrinsic::riscv_vssseg2_mask:
2701 case Intrinsic::riscv_vssseg3_mask:
2702 case Intrinsic::riscv_vssseg4_mask:
2703 case Intrinsic::riscv_vssseg5_mask:
2704 case Intrinsic::riscv_vssseg6_mask:
2705 case Intrinsic::riscv_vssseg7_mask:
2706 case Intrinsic::riscv_vssseg8_mask: {
2707 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2708 /*IsStrided*/ true);
2709 return;
2710 }
2711 case Intrinsic::riscv_vsoxseg2:
2712 case Intrinsic::riscv_vsoxseg3:
2713 case Intrinsic::riscv_vsoxseg4:
2714 case Intrinsic::riscv_vsoxseg5:
2715 case Intrinsic::riscv_vsoxseg6:
2716 case Intrinsic::riscv_vsoxseg7:
2717 case Intrinsic::riscv_vsoxseg8:
2718 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2719 /*IsOrdered*/ true);
2720 return;
2721 case Intrinsic::riscv_vsuxseg2:
2722 case Intrinsic::riscv_vsuxseg3:
2723 case Intrinsic::riscv_vsuxseg4:
2724 case Intrinsic::riscv_vsuxseg5:
2725 case Intrinsic::riscv_vsuxseg6:
2726 case Intrinsic::riscv_vsuxseg7:
2727 case Intrinsic::riscv_vsuxseg8:
2728 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2729 /*IsOrdered*/ false);
2730 return;
2731 case Intrinsic::riscv_vsoxseg2_mask:
2732 case Intrinsic::riscv_vsoxseg3_mask:
2733 case Intrinsic::riscv_vsoxseg4_mask:
2734 case Intrinsic::riscv_vsoxseg5_mask:
2735 case Intrinsic::riscv_vsoxseg6_mask:
2736 case Intrinsic::riscv_vsoxseg7_mask:
2737 case Intrinsic::riscv_vsoxseg8_mask:
2738 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2739 /*IsOrdered*/ true);
2740 return;
2741 case Intrinsic::riscv_vsuxseg2_mask:
2742 case Intrinsic::riscv_vsuxseg3_mask:
2743 case Intrinsic::riscv_vsuxseg4_mask:
2744 case Intrinsic::riscv_vsuxseg5_mask:
2745 case Intrinsic::riscv_vsuxseg6_mask:
2746 case Intrinsic::riscv_vsuxseg7_mask:
2747 case Intrinsic::riscv_vsuxseg8_mask:
2748 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2749 /*IsOrdered*/ false);
2750 return;
2751 case Intrinsic::riscv_vsoxei:
2752 case Intrinsic::riscv_vsoxei_mask:
2753 case Intrinsic::riscv_vsuxei:
2754 case Intrinsic::riscv_vsuxei_mask: {
2755 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
2756 IntNo == Intrinsic::riscv_vsuxei_mask;
2757 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
2758 IntNo == Intrinsic::riscv_vsoxei_mask;
2759
2760 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2761 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2762
2763 unsigned CurOp = 2;
2764 SmallVector<SDValue, 8> Operands;
2765 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2766
2767 MVT IndexVT;
2768 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2769 /*IsStridedOrIndexed*/ true, Operands,
2770 /*IsLoad=*/false, IndexVT: &IndexVT);
2771
2772 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2773 "Element count mismatch");
2774
2775 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2776 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2777 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2778 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2779 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2780 "index values when XLEN=32");
2781 }
2782 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
2783 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW,
2784 LMUL: static_cast<unsigned>(LMUL), IndexLMUL: static_cast<unsigned>(IndexLMUL));
2785 MachineSDNode *Store =
2786 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2787
2788 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2789
2790 ReplaceNode(F: Node, T: Store);
2791 return;
2792 }
2793 case Intrinsic::riscv_vsm:
2794 case Intrinsic::riscv_vse:
2795 case Intrinsic::riscv_vse_mask:
2796 case Intrinsic::riscv_vsse:
2797 case Intrinsic::riscv_vsse_mask: {
2798 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
2799 IntNo == Intrinsic::riscv_vsse_mask;
2800 bool IsStrided =
2801 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
2802
2803 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2804 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2805
2806 unsigned CurOp = 2;
2807 SmallVector<SDValue, 8> Operands;
2808 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2809
2810 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2811 Operands);
2812
2813 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2814 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
2815 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2816 MachineSDNode *Store =
2817 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2818 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2819
2820 ReplaceNode(F: Node, T: Store);
2821 return;
2822 }
2823 case Intrinsic::riscv_sf_vc_x_se:
2824 case Intrinsic::riscv_sf_vc_i_se:
2825 selectSF_VC_X_SE(Node);
2826 return;
2827 case Intrinsic::riscv_sf_vlte8:
2828 case Intrinsic::riscv_sf_vlte16:
2829 case Intrinsic::riscv_sf_vlte32:
2830 case Intrinsic::riscv_sf_vlte64: {
2831 unsigned Log2SEW;
2832 unsigned PseudoInst;
2833 switch (IntNo) {
2834 case Intrinsic::riscv_sf_vlte8:
2835 PseudoInst = RISCV::PseudoSF_VLTE8;
2836 Log2SEW = 3;
2837 break;
2838 case Intrinsic::riscv_sf_vlte16:
2839 PseudoInst = RISCV::PseudoSF_VLTE16;
2840 Log2SEW = 4;
2841 break;
2842 case Intrinsic::riscv_sf_vlte32:
2843 PseudoInst = RISCV::PseudoSF_VLTE32;
2844 Log2SEW = 5;
2845 break;
2846 case Intrinsic::riscv_sf_vlte64:
2847 PseudoInst = RISCV::PseudoSF_VLTE64;
2848 Log2SEW = 6;
2849 break;
2850 }
2851
2852 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
2853 SDValue TWidenOp = CurDAG->getTargetConstant(Val: 1, DL, VT: XLenVT);
2854 SDValue Operands[] = {Node->getOperand(Num: 2),
2855 Node->getOperand(Num: 3),
2856 Node->getOperand(Num: 4),
2857 SEWOp,
2858 TWidenOp,
2859 Node->getOperand(Num: 0)};
2860
2861 MachineSDNode *TileLoad =
2862 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2863 CurDAG->setNodeMemRefs(N: TileLoad,
2864 NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2865
2866 ReplaceNode(F: Node, T: TileLoad);
2867 return;
2868 }
2869 case Intrinsic::riscv_sf_mm_s_s:
2870 case Intrinsic::riscv_sf_mm_s_u:
2871 case Intrinsic::riscv_sf_mm_u_s:
2872 case Intrinsic::riscv_sf_mm_u_u:
2873 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2874 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2875 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2876 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
2877 case Intrinsic::riscv_sf_mm_f_f: {
2878 bool HasFRM = false;
2879 unsigned PseudoInst;
2880 switch (IntNo) {
2881 case Intrinsic::riscv_sf_mm_s_s:
2882 PseudoInst = RISCV::PseudoSF_MM_S_S;
2883 break;
2884 case Intrinsic::riscv_sf_mm_s_u:
2885 PseudoInst = RISCV::PseudoSF_MM_S_U;
2886 break;
2887 case Intrinsic::riscv_sf_mm_u_s:
2888 PseudoInst = RISCV::PseudoSF_MM_U_S;
2889 break;
2890 case Intrinsic::riscv_sf_mm_u_u:
2891 PseudoInst = RISCV::PseudoSF_MM_U_U;
2892 break;
2893 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2894 PseudoInst = RISCV::PseudoSF_MM_E5M2_E5M2;
2895 HasFRM = true;
2896 break;
2897 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2898 PseudoInst = RISCV::PseudoSF_MM_E5M2_E4M3;
2899 HasFRM = true;
2900 break;
2901 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2902 PseudoInst = RISCV::PseudoSF_MM_E4M3_E5M2;
2903 HasFRM = true;
2904 break;
2905 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
2906 PseudoInst = RISCV::PseudoSF_MM_E4M3_E4M3;
2907 HasFRM = true;
2908 break;
2909 case Intrinsic::riscv_sf_mm_f_f:
2910 if (Node->getOperand(Num: 3).getValueType().getScalarType() == MVT::bf16)
2911 PseudoInst = RISCV::PseudoSF_MM_F_F_ALT;
2912 else
2913 PseudoInst = RISCV::PseudoSF_MM_F_F;
2914 HasFRM = true;
2915 break;
2916 }
2917 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
2918 SDValue Op1 = Node->getOperand(Num: 3);
2919 SDValue Op2 = Node->getOperand(Num: 4);
2920 MVT VT = Op1->getSimpleValueType(ResNo: 0);
2921 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2922 SDValue TmOp = Node->getOperand(Num: 5);
2923 SDValue TnOp = Node->getOperand(Num: 6);
2924 SDValue TkOp = Node->getOperand(Num: 7);
2925 SDValue TWidenOp = Node->getOperand(Num: 8);
2926 SDValue Chain = Node->getOperand(Num: 0);
2927
2928 // sf.mm.f.f with sew=32, twiden=2 is invalid
2929 if (IntNo == Intrinsic::riscv_sf_mm_f_f && Log2SEW == 5 &&
2930 TWidenOp->getAsZExtVal() == 2)
2931 reportFatalUsageError(reason: "sf.mm.f.f doesn't support (sew=32, twiden=2)");
2932
2933 SmallVector<SDValue, 10> Operands(
2934 {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Op1, Op2});
2935 if (HasFRM)
2936 Operands.push_back(
2937 Elt: CurDAG->getTargetConstant(Val: RISCVFPRndMode::DYN, DL, VT: XLenVT));
2938 Operands.append(IL: {TmOp, TnOp, TkOp,
2939 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT), TWidenOp,
2940 Chain});
2941
2942 auto *NewNode =
2943 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2944
2945 ReplaceNode(F: Node, T: NewNode);
2946 return;
2947 }
2948 case Intrinsic::riscv_sf_vtzero_t: {
2949 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
2950 SDValue Tm = Node->getOperand(Num: 3);
2951 SDValue Tn = Node->getOperand(Num: 4);
2952 SDValue Log2SEW = Node->getOperand(Num: 5);
2953 SDValue TWiden = Node->getOperand(Num: 6);
2954 SDValue Chain = Node->getOperand(Num: 0);
2955 auto *NewNode = CurDAG->getMachineNode(
2956 Opcode: RISCV::PseudoSF_VTZERO_T, dl: DL, VTs: Node->getVTList(),
2957 Ops: {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Tm, Tn, Log2SEW,
2958 TWiden, Chain});
2959
2960 ReplaceNode(F: Node, T: NewNode);
2961 return;
2962 }
2963 }
2964 break;
2965 }
2966 case ISD::BITCAST: {
2967 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
2968 // Just drop bitcasts between vectors if both are fixed or both are
2969 // scalable.
2970 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
2971 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
2972 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
2973 CurDAG->RemoveDeadNode(N: Node);
2974 return;
2975 }
2976 if (Subtarget->hasStdExtP()) {
2977 bool Is32BitCast =
2978 (VT == MVT::i32 && (SrcVT == MVT::v4i8 || SrcVT == MVT::v2i16)) ||
2979 (SrcVT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
2980 bool Is64BitCast =
2981 (VT == MVT::i64 && (SrcVT == MVT::v8i8 || SrcVT == MVT::v4i16 ||
2982 SrcVT == MVT::v2i32)) ||
2983 (SrcVT == MVT::i64 &&
2984 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
2985 if (Is32BitCast || Is64BitCast) {
2986 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
2987 CurDAG->RemoveDeadNode(N: Node);
2988 return;
2989 }
2990 }
2991 break;
2992 }
2993 case ISD::SPLAT_VECTOR: {
2994 if (!Subtarget->hasStdExtP())
2995 break;
2996 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0))) {
2997 bool IsDoubleWide = Subtarget->isPExtPackedDoubleType(VT);
2998
2999 if (ConstNode->isZero()) {
3000 MCPhysReg X0Reg = IsDoubleWide ? RISCV::X0_Pair : RISCV::X0;
3001 SDValue New =
3002 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: X0Reg, VT);
3003 ReplaceNode(F: Node, T: New.getNode());
3004 return;
3005 }
3006
3007 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3008 APInt Val = ConstNode->getAPIntValue().trunc(width: EltSize);
3009
3010 // Use LI for all ones since it can be compressed to c.li.
3011 if (Val.isAllOnes() && !IsDoubleWide) {
3012 SDNode *NewNode = CurDAG->getMachineNode(
3013 Opcode: RISCV::ADDI, dl: DL, VT, Op1: CurDAG->getRegister(Reg: RISCV::X0, VT),
3014 Op2: CurDAG->getAllOnesConstant(DL, VT: XLenVT, /*IsTarget=*/true));
3015 ReplaceNode(F: Node, T: NewNode);
3016 return;
3017 }
3018
3019 // Find the smallest splat.
3020 if (Val.getBitWidth() > 16 && Val.isSplat(SplatSizeInBits: 16))
3021 Val = Val.trunc(width: 16);
3022 if (Val.getBitWidth() > 8 && Val.isSplat(SplatSizeInBits: 8))
3023 Val = Val.trunc(width: 8);
3024
3025 EltSize = Val.getBitWidth();
3026 int64_t Imm = Val.getSExtValue();
3027
3028 unsigned Opc = 0;
3029 if (EltSize == 8) {
3030 Opc = IsDoubleWide ? RISCV::PLI_DB : RISCV::PLI_B;
3031 } else if (EltSize == 16 && isInt<10>(x: Imm)) {
3032 Opc = IsDoubleWide ? RISCV::PLI_DH : RISCV::PLI_H;
3033 } else if (!IsDoubleWide && EltSize == 32 && isInt<10>(x: Imm)) {
3034 Opc = RISCV::PLI_W;
3035 } else if (EltSize == 16 && isShiftedInt<10, 6>(x: Imm)) {
3036 Opc = IsDoubleWide ? RISCV::PLUI_DH : RISCV::PLUI_H;
3037 Imm = Imm >> 6;
3038 } else if (!IsDoubleWide && EltSize == 32 && isShiftedInt<10, 22>(x: Imm)) {
3039 Opc = RISCV::PLUI_W;
3040 Imm = Imm >> 22;
3041 }
3042
3043 if (Opc) {
3044 SDNode *NewNode = CurDAG->getMachineNode(
3045 Opcode: Opc, dl: DL, VT, Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT: XLenVT));
3046 ReplaceNode(F: Node, T: NewNode);
3047 return;
3048 }
3049 }
3050
3051 break;
3052 }
3053 case ISD::SCALAR_TO_VECTOR:
3054 if (Subtarget->hasStdExtP()) {
3055 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3056 if ((VT == MVT::v2i32 && SrcVT == MVT::i64) ||
3057 (VT == MVT::v4i8 && SrcVT == MVT::i32)) {
3058 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3059 CurDAG->RemoveDeadNode(N: Node);
3060 return;
3061 }
3062 }
3063 break;
3064 case ISD::INSERT_SUBVECTOR:
3065 case RISCVISD::TUPLE_INSERT: {
3066 SDValue V = Node->getOperand(Num: 0);
3067 SDValue SubV = Node->getOperand(Num: 1);
3068 SDLoc DL(SubV);
3069 auto Idx = Node->getConstantOperandVal(Num: 2);
3070 MVT SubVecVT = SubV.getSimpleValueType();
3071
3072 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3073 MVT SubVecContainerVT = SubVecVT;
3074 // Establish the correct scalable-vector types for any fixed-length type.
3075 if (SubVecVT.isFixedLengthVector()) {
3076 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT: SubVecVT);
3077 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
3078 [[maybe_unused]] bool ExactlyVecRegSized =
3079 Subtarget->expandVScale(X: SubVecVT.getSizeInBits())
3080 .isKnownMultipleOf(RHS: Subtarget->expandVScale(X: VecRegSize));
3081 assert(isPowerOf2_64(Subtarget->expandVScale(SubVecVT.getSizeInBits())
3082 .getKnownMinValue()));
3083 assert(Idx == 0 && (ExactlyVecRegSized || V.isUndef()));
3084 }
3085 MVT ContainerVT = VT;
3086 if (VT.isFixedLengthVector())
3087 ContainerVT = TLI.getContainerForFixedLengthVector(VT);
3088
3089 const auto *TRI = Subtarget->getRegisterInfo();
3090 unsigned SubRegIdx;
3091 std::tie(args&: SubRegIdx, args&: Idx) =
3092 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3093 VecVT: ContainerVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3094
3095 // If the Idx hasn't been completely eliminated then this is a subvector
3096 // insert which doesn't naturally align to a vector register. These must
3097 // be handled using instructions to manipulate the vector registers.
3098 if (Idx != 0)
3099 break;
3100
3101 RISCVVType::VLMUL SubVecLMUL =
3102 RISCVTargetLowering::getLMUL(VT: SubVecContainerVT);
3103 [[maybe_unused]] bool IsSubVecPartReg =
3104 SubVecLMUL == RISCVVType::VLMUL::LMUL_F2 ||
3105 SubVecLMUL == RISCVVType::VLMUL::LMUL_F4 ||
3106 SubVecLMUL == RISCVVType::VLMUL::LMUL_F8;
3107 assert((V.getValueType().isRISCVVectorTuple() || !IsSubVecPartReg ||
3108 V.isUndef()) &&
3109 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
3110 "the subvector is smaller than a full-sized register");
3111
3112 // If we haven't set a SubRegIdx, then we must be going between
3113 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
3114 if (SubRegIdx == RISCV::NoSubRegister) {
3115 unsigned InRegClassID =
3116 RISCVTargetLowering::getRegClassIDForVecVT(VT: ContainerVT);
3117 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3118 InRegClassID &&
3119 "Unexpected subvector extraction");
3120 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3121 SDNode *NewNode = CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
3122 dl: DL, VT, Op1: SubV, Op2: RC);
3123 ReplaceNode(F: Node, T: NewNode);
3124 return;
3125 }
3126
3127 SDValue Insert = CurDAG->getTargetInsertSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V, Subreg: SubV);
3128 ReplaceNode(F: Node, T: Insert.getNode());
3129 return;
3130 }
3131 case ISD::EXTRACT_SUBVECTOR:
3132 case RISCVISD::TUPLE_EXTRACT: {
3133 if (Subtarget->hasStdExtP())
3134 break;
3135
3136 SDValue V = Node->getOperand(Num: 0);
3137 auto Idx = Node->getConstantOperandVal(Num: 1);
3138 MVT InVT = V.getSimpleValueType();
3139
3140 SDLoc DL(V);
3141
3142 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3143 MVT SubVecContainerVT = VT;
3144 // Establish the correct scalable-vector types for any fixed-length type.
3145 if (VT.isFixedLengthVector()) {
3146 assert(Idx == 0);
3147 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
3148 }
3149 if (InVT.isFixedLengthVector())
3150 InVT = TLI.getContainerForFixedLengthVector(VT: InVT);
3151
3152 const auto *TRI = Subtarget->getRegisterInfo();
3153 unsigned SubRegIdx;
3154 std::tie(args&: SubRegIdx, args&: Idx) =
3155 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3156 VecVT: InVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3157
3158 // If the Idx hasn't been completely eliminated then this is a subvector
3159 // extract which doesn't naturally align to a vector register. These must
3160 // be handled using instructions to manipulate the vector registers.
3161 if (Idx != 0)
3162 break;
3163
3164 // If we haven't set a SubRegIdx, then we must be going between
3165 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
3166 if (SubRegIdx == RISCV::NoSubRegister) {
3167 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: InVT);
3168 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3169 InRegClassID &&
3170 "Unexpected subvector extraction");
3171 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3172 SDNode *NewNode =
3173 CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT, Op1: V, Op2: RC);
3174 ReplaceNode(F: Node, T: NewNode);
3175 return;
3176 }
3177
3178 SDValue Extract = CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V);
3179 ReplaceNode(F: Node, T: Extract.getNode());
3180 return;
3181 }
3182 case RISCVISD::VMV_S_X_VL:
3183 case RISCVISD::VFMV_S_F_VL:
3184 case RISCVISD::VMV_V_X_VL:
3185 case RISCVISD::VFMV_V_F_VL: {
3186 // Try to match splat of a scalar load to a strided load with stride of x0.
3187 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
3188 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
3189 if (!Node->getOperand(Num: 0).isUndef())
3190 break;
3191 SDValue Src = Node->getOperand(Num: 1);
3192 auto *Ld = dyn_cast<LoadSDNode>(Val&: Src);
3193 // Can't fold load update node because the second
3194 // output is used so that load update node can't be removed.
3195 if (!Ld || Ld->isIndexed())
3196 break;
3197 EVT MemVT = Ld->getMemoryVT();
3198 // The memory VT should be the same size as the element type.
3199 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
3200 break;
3201 if (!IsProfitableToFold(N: Src, U: Node, Root: Node) ||
3202 !IsLegalToFold(N: Src, U: Node, Root: Node, OptLevel: TM.getOptLevel()))
3203 break;
3204
3205 SDValue VL;
3206 if (IsScalarMove) {
3207 // We could deal with more VL if we update the VSETVLI insert pass to
3208 // avoid introducing more VSETVLI.
3209 if (!isOneConstant(V: Node->getOperand(Num: 2)))
3210 break;
3211 selectVLOp(N: Node->getOperand(Num: 2), VL);
3212 } else
3213 selectVLOp(N: Node->getOperand(Num: 2), VL);
3214
3215 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3216 SDValue SEW = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
3217
3218 // If VL=1, then we don't need to do a strided load and can just do a
3219 // regular load.
3220 bool IsStrided = !isOneConstant(V: VL);
3221
3222 // Only do a strided load if we have optimized zero-stride vector load.
3223 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
3224 break;
3225
3226 SmallVector<SDValue> Operands = {
3227 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT), 0),
3228 Ld->getBasePtr()};
3229 if (IsStrided)
3230 Operands.push_back(Elt: CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT));
3231 uint64_t Policy = RISCVVType::MASK_AGNOSTIC | RISCVVType::TAIL_AGNOSTIC;
3232 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
3233 Operands.append(IL: {VL, SEW, PolicyOp, Ld->getChain()});
3234
3235 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
3236 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
3237 /*IsMasked*/ Masked: false, Strided: IsStrided, /*FF*/ false,
3238 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
3239 MachineSDNode *Load =
3240 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, ResultTys: {VT, MVT::Other}, Ops: Operands);
3241 // Update the chain.
3242 ReplaceUses(F: Src.getValue(R: 1), T: SDValue(Load, 1));
3243 // Record the mem-refs
3244 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {Ld->getMemOperand()});
3245 // Replace the splat with the vlse.
3246 ReplaceNode(F: Node, T: Load);
3247 return;
3248 }
3249 case RISCVISD::LPAD_CALL:
3250 case RISCVISD::LPAD_CALL_INDIRECT: {
3251 bool IsIndirect = Opcode == RISCVISD::LPAD_CALL_INDIRECT;
3252 unsigned PseudoOpc = IsIndirect ? RISCV::PseudoCALLIndirectLpadAlign
3253 : RISCV::PseudoCALLLpadAlign;
3254
3255 uint32_t LpadLabel = 0;
3256 if (PreferredLandingPadLabel.getNumOccurrences() > 0) {
3257 if (!isUInt<20>(x: PreferredLandingPadLabel))
3258 report_fatal_error(reason: "riscv-landing-pad-label=<val>, <val> needs to fit "
3259 "in unsigned 20-bits");
3260 LpadLabel = PreferredLandingPadLabel;
3261 }
3262
3263 // Preserve the argument-register and register-mask operands, between
3264 // Callee and the optional glue, so the pseudo call still reports its
3265 // call-preserved mask to the register allocator.
3266 SmallVector<SDValue, 8> Ops;
3267 Ops.push_back(Elt: Node->getOperand(Num: 1));
3268 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: LpadLabel, DL, VT: XLenVT));
3269
3270 unsigned NumOps = Node->getNumOperands();
3271 bool HasGlue = Node->getGluedNode() != nullptr;
3272 unsigned RegOperandsEnd = HasGlue ? NumOps - 1 : NumOps;
3273 for (unsigned I = 2; I != RegOperandsEnd; ++I)
3274 Ops.push_back(Elt: Node->getOperand(Num: I));
3275
3276 Ops.push_back(Elt: Node->getOperand(Num: 0));
3277 if (HasGlue)
3278 Ops.push_back(Elt: Node->getOperand(Num: NumOps - 1));
3279
3280 ReplaceNode(F: Node,
3281 T: CurDAG->getMachineNode(Opcode: PseudoOpc, dl: DL, VTs: Node->getVTList(), Ops));
3282 return;
3283 }
3284 case ISD::PREFETCH:
3285 // MIPS's prefetch instruction already encodes the hint within the
3286 // instruction itself, so no extra NTL hint is needed.
3287 if (Subtarget->hasVendorXMIPSCBOP())
3288 break;
3289
3290 unsigned Locality = Node->getConstantOperandVal(Num: 3);
3291 if (Locality > 2)
3292 break;
3293
3294 auto *LoadStoreMem = cast<MemSDNode>(Val: Node);
3295 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
3296 MMO->setFlags(MachineMemOperand::MONonTemporal);
3297
3298 int NontemporalLevel = 0;
3299 switch (Locality) {
3300 case 0:
3301 NontemporalLevel = 3; // NTL.ALL
3302 break;
3303 case 1:
3304 NontemporalLevel = 1; // NTL.PALL
3305 break;
3306 case 2:
3307 NontemporalLevel = 0; // NTL.P1
3308 break;
3309 default:
3310 llvm_unreachable("unexpected locality value.");
3311 }
3312
3313 if (NontemporalLevel & 0b1)
3314 MMO->setFlags(MONontemporalBit0);
3315 if (NontemporalLevel & 0b10)
3316 MMO->setFlags(MONontemporalBit1);
3317 break;
3318 }
3319
3320 // Select the default instruction.
3321 SelectCode(N: Node);
3322}
3323
3324bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
3325 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
3326 std::vector<SDValue> &OutOps) {
3327 // Always produce a register and immediate operand, as expected by
3328 // RISCVAsmPrinter::PrintAsmMemoryOperand.
3329 switch (ConstraintID) {
3330 case InlineAsm::ConstraintCode::o:
3331 case InlineAsm::ConstraintCode::m: {
3332 SDValue Op0, Op1;
3333 [[maybe_unused]] bool Found = SelectAddrRegImm(Addr: Op, Base&: Op0, Offset&: Op1);
3334 assert(Found && "SelectAddrRegImm should always succeed");
3335 OutOps.push_back(x: Op0);
3336 OutOps.push_back(x: Op1);
3337 return false;
3338 }
3339 case InlineAsm::ConstraintCode::A:
3340 OutOps.push_back(x: Op);
3341 OutOps.push_back(
3342 x: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget->getXLenVT()));
3343 return false;
3344 default:
3345 report_fatal_error(reason: "Unexpected asm memory constraint " +
3346 InlineAsm::getMemConstraintName(C: ConstraintID));
3347 }
3348
3349 return true;
3350}
3351
3352bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
3353 SDValue &Offset) {
3354 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Addr)) {
3355 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT: Subtarget->getXLenVT());
3356 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: Subtarget->getXLenVT());
3357 return true;
3358 }
3359
3360 return false;
3361}
3362
3363// Fold constant addresses.
3364static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
3365 const MVT VT, const RISCVSubtarget *Subtarget,
3366 SDValue Addr, SDValue &Base, SDValue &Offset,
3367 bool IsPrefetch = false) {
3368 if (!isa<ConstantSDNode>(Val: Addr))
3369 return false;
3370
3371 int64_t CVal = cast<ConstantSDNode>(Val&: Addr)->getSExtValue();
3372
3373 // If the constant is a simm12, we can fold the whole constant and use X0 as
3374 // the base. If the constant can be materialized with LUI+simm12, use LUI as
3375 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
3376 int64_t Lo12 = SignExtend64<12>(x: CVal);
3377 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
3378 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
3379 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3380 return false;
3381 if (Hi) {
3382 int64_t Hi20 = (Hi >> 12) & 0xfffff;
3383 Base = SDValue(
3384 CurDAG->getMachineNode(Opcode: RISCV::LUI, dl: DL, VT,
3385 Op1: CurDAG->getTargetConstant(Val: Hi20, DL, VT)),
3386 0);
3387 } else {
3388 Base = CurDAG->getRegister(Reg: RISCV::X0, VT);
3389 }
3390 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3391 return true;
3392 }
3393
3394 // Ask how constant materialization would handle this constant.
3395 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
3396
3397 // If the last instruction would be an ADDI, we can fold its immediate and
3398 // emit the rest of the sequence as the base.
3399 if (Seq.back().getOpcode() != RISCV::ADDI)
3400 return false;
3401 Lo12 = Seq.back().getImm();
3402 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3403 return false;
3404
3405 // Drop the last instruction.
3406 Seq.pop_back();
3407 assert(!Seq.empty() && "Expected more instructions in sequence");
3408
3409 Base = selectImmSeq(CurDAG, DL, VT, Seq);
3410 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3411 return true;
3412}
3413
3414// Is this ADD instruction only used as the base pointer of scalar loads and
3415// stores?
3416static bool isWorthFoldingAdd(SDValue Add) {
3417 for (auto *User : Add->users()) {
3418 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
3419 User->getOpcode() != RISCVISD::LD_RV32 &&
3420 User->getOpcode() != RISCVISD::SD_RV32 &&
3421 User->getOpcode() != ISD::ATOMIC_LOAD &&
3422 User->getOpcode() != ISD::ATOMIC_STORE)
3423 return false;
3424 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3425 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
3426 VT != MVT::f64)
3427 return false;
3428 // Don't allow stores of the value. It must be used as the address.
3429 if (User->getOpcode() == ISD::STORE &&
3430 cast<StoreSDNode>(Val: User)->getValue() == Add)
3431 return false;
3432 if (User->getOpcode() == ISD::ATOMIC_STORE &&
3433 cast<AtomicSDNode>(Val: User)->getVal() == Add)
3434 return false;
3435 if (User->getOpcode() == RISCVISD::SD_RV32 &&
3436 (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add))
3437 return false;
3438 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
3439 return false;
3440 }
3441
3442 return true;
3443}
3444
3445bool isRegImmLoadOrStore(SDNode *User, SDValue Add) {
3446 switch (User->getOpcode()) {
3447 default:
3448 return false;
3449 case ISD::LOAD:
3450 case RISCVISD::LD_RV32:
3451 case ISD::ATOMIC_LOAD:
3452 break;
3453 case ISD::STORE:
3454 // Don't allow stores of Add. It must only be used as the address.
3455 if (cast<StoreSDNode>(Val: User)->getValue() == Add)
3456 return false;
3457 break;
3458 case RISCVISD::SD_RV32:
3459 // Don't allow stores of Add. It must only be used as the address.
3460 if (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add)
3461 return false;
3462 break;
3463 case ISD::ATOMIC_STORE:
3464 // Don't allow stores of Add. It must only be used as the address.
3465 if (cast<AtomicSDNode>(Val: User)->getVal() == Add)
3466 return false;
3467 break;
3468 }
3469
3470 return true;
3471}
3472
3473// To prevent SelectAddrRegImm from folding offsets that conflict with the
3474// fusion of PseudoMovAddr, check if the offset of every use of a given address
3475// is within the alignment.
3476bool RISCVDAGToDAGISel::areOffsetsWithinAlignment(SDValue Addr,
3477 Align Alignment) {
3478 assert(Addr->getOpcode() == RISCVISD::ADD_LO);
3479 for (auto *User : Addr->users()) {
3480 // If the user is a load or store, then the offset is 0 which is always
3481 // within alignment.
3482 if (isRegImmLoadOrStore(User, Add: Addr))
3483 continue;
3484
3485 if (CurDAG->isBaseWithConstantOffset(Op: SDValue(User, 0))) {
3486 int64_t CVal = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
3487 if (!isInt<12>(x: CVal) || Alignment <= CVal)
3488 return false;
3489
3490 // Make sure all uses are foldable load/stores.
3491 for (auto *AddUser : User->users())
3492 if (!isRegImmLoadOrStore(User: AddUser, Add: SDValue(User, 0)))
3493 return false;
3494
3495 continue;
3496 }
3497
3498 return false;
3499 }
3500
3501 return true;
3502}
3503
3504bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
3505 SDValue &Offset) {
3506 if (SelectAddrFrameIndex(Addr, Base, Offset))
3507 return true;
3508
3509 SDLoc DL(Addr);
3510 MVT VT = Addr.getSimpleValueType();
3511
3512 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
3513 bool CanFold = true;
3514 // Unconditionally fold if operand 1 is not a global address (e.g.
3515 // externsymbol)
3516 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val: Addr.getOperand(i: 1))) {
3517 const DataLayout &DL = CurDAG->getDataLayout();
3518 Align Alignment = commonAlignment(
3519 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3520 if (!areOffsetsWithinAlignment(Addr, Alignment))
3521 CanFold = false;
3522 }
3523 if (CanFold) {
3524 Base = Addr.getOperand(i: 0);
3525 Offset = Addr.getOperand(i: 1);
3526 return true;
3527 }
3528 }
3529
3530 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3531 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3532 if (isInt<12>(x: CVal)) {
3533 Base = Addr.getOperand(i: 0);
3534 if (Base.getOpcode() == RISCVISD::ADD_LO) {
3535 SDValue LoOperand = Base.getOperand(i: 1);
3536 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: LoOperand)) {
3537 // If the Lo in (ADD_LO hi, lo) is a global variable's address
3538 // (its low part, really), then we can rely on the alignment of that
3539 // variable to provide a margin of safety before low part can overflow
3540 // the 12 bits of the load/store offset. Check if CVal falls within
3541 // that margin; if so (low part + CVal) can't overflow.
3542 const DataLayout &DL = CurDAG->getDataLayout();
3543 Align Alignment = commonAlignment(
3544 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3545 if ((CVal == 0 || Alignment > CVal) &&
3546 areOffsetsWithinAlignment(Addr: Base, Alignment)) {
3547 int64_t CombinedOffset = CVal + GA->getOffset();
3548 Base = Base.getOperand(i: 0);
3549 Offset = CurDAG->getTargetGlobalAddress(
3550 GV: GA->getGlobal(), DL: SDLoc(LoOperand), VT: LoOperand.getValueType(),
3551 offset: CombinedOffset, TargetFlags: GA->getTargetFlags());
3552 return true;
3553 }
3554 }
3555 }
3556
3557 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3558 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3559 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3560 return true;
3561 }
3562 }
3563
3564 // Handle ADD with large immediates.
3565 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3566 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3567 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3568
3569 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
3570 // an ADDI for part of the offset and fold the rest into the load/store.
3571 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
3572 if (CVal >= -4096 && CVal <= 4094) {
3573 int64_t Adj = CVal < 0 ? -2048 : 2047;
3574 Base = SDValue(
3575 CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3576 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3577 0);
3578 Offset = CurDAG->getSignedTargetConstant(Val: CVal - Adj, DL, VT);
3579 return true;
3580 }
3581
3582 // For larger immediates, we might be able to save one instruction from
3583 // constant materialization by folding the Lo12 bits of the immediate into
3584 // the address. We should only do this if the ADD is only used by loads and
3585 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
3586 // separately with the full materialized immediate creating extra
3587 // instructions.
3588 if (isWorthFoldingAdd(Add: Addr) &&
3589 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3590 Offset, /*IsPrefetch=*/false)) {
3591 // Insert an ADD instruction with the materialized Hi52 bits.
3592 Base = SDValue(
3593 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3594 0);
3595 return true;
3596 }
3597 }
3598
3599 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3600 /*IsPrefetch=*/false))
3601 return true;
3602
3603 Base = Addr;
3604 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3605 return true;
3606}
3607
3608/// Similar to SelectAddrRegImm, except that the offset is a 26-bit signed
3609/// immediate. This is used by the Qualcomm Xqcilo large offset load/store
3610/// instructions (qc.e.lw/qc.e.sw), whose offset field is 26 bits wide.
3611/// Only matches offsets that do not fit a 12-bit signed immediate, so that
3612/// offsets in the simm12 range keep using the shorter (and possibly
3613/// compressible) standard load/store instructions.
3614bool RISCVDAGToDAGISel::SelectAddrRegImm26(SDValue Addr, SDValue &Base,
3615 SDValue &Offset) {
3616 SDLoc DL(Addr);
3617 MVT VT = Addr.getSimpleValueType();
3618
3619 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3620 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3621 // Fold a 26-bit (but not 12-bit) signed offset directly into the
3622 // load/store.
3623 if (isInt<26>(x: CVal) && !isInt<12>(x: CVal)) {
3624 Base = Addr.getOperand(i: 0);
3625 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3626 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3627 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3628 return true;
3629 }
3630 }
3631
3632 // The offset is just outside the 26-bit range. Split off a small (simm12)
3633 // adjustment with a plain ADDI and fold the remaining 26-bit offset into the
3634 // load/store. A plain ADDI is used (rather than the wide
3635 // qc.e.addi/qc.e.addai) because the adjustment fits simm12: this keeps it a
3636 // short, compressible (c.addi) instruction and is available without Xqcilia.
3637 //
3638 // Skip the split if the address is used other than as a foldable load/store
3639 // base. `isWorthFoldingAdd()` returns true when every user of the add node is
3640 // a scalar load/store using it as an address operand. If it return false, it
3641 // means that some use consumes the add result as a value (e.g. it feeds
3642 // another add, is a stored value, is used in arithmetic) and that use forces
3643 // the add to be materialized into a register.
3644 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)) &&
3645 isWorthFoldingAdd(Add: Addr)) {
3646 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3647 if (!isInt<26>(x: CVal)) {
3648 // check if lw in lui + add + lw combination can be compressed.
3649 // The check here purely based on the immediate value and hopes that
3650 // register allocator would assign a register from a GPRC set so that the
3651 // instruction can get compressed.
3652 bool IsLwCompressable = isShiftedUInt<5, 2>(x: CVal & ((1 << 12) - 1));
3653
3654 int64_t Imm26 = CVal < 0 ? minIntN(N: 26) : maxIntN(N: 26);
3655 int64_t Adj = CVal - Imm26;
3656 // If Adj fits within 6-bits, then both combinations will take 8 bytes
3657 // however c.addi + qc.e.lw/sw will take 1 less cycle. Also, if lw is not
3658 // compressable then both combination would take 10 bytes but again
3659 // addi + qc.e.lw/sw will take 1 less cycle.
3660 if (isInt<6>(x: Adj) || (isInt<12>(x: Adj) && !IsLwCompressable)) {
3661 Base = SDValue(CurDAG->getMachineNode(
3662 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3663 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3664 0);
3665 Offset = CurDAG->getSignedTargetConstant(Val: Imm26, DL, VT);
3666 return true;
3667 }
3668 }
3669 }
3670
3671 // Don't match: let the standard addressing modes handle it.
3672 return false;
3673}
3674
3675/// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
3676bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
3677 SDValue &Offset) {
3678 if (SelectAddrFrameIndex(Addr, Base, Offset))
3679 return true;
3680
3681 SDLoc DL(Addr);
3682 MVT VT = Addr.getSimpleValueType();
3683
3684 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3685 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3686 if (isUInt<9>(x: CVal)) {
3687 Base = Addr.getOperand(i: 0);
3688
3689 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3690 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3691 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3692 return true;
3693 }
3694 }
3695
3696 Base = Addr;
3697 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3698 return true;
3699}
3700
3701/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
3702/// Offset should be all zeros.
3703bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
3704 SDValue &Offset) {
3705 if (SelectAddrFrameIndex(Addr, Base, Offset))
3706 return true;
3707
3708 SDLoc DL(Addr);
3709 MVT VT = Addr.getSimpleValueType();
3710
3711 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3712 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3713 if (isInt<12>(x: CVal)) {
3714 Base = Addr.getOperand(i: 0);
3715
3716 // Early-out if not a valid offset.
3717 if ((CVal & 0b11111) != 0) {
3718 Base = Addr;
3719 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3720 return true;
3721 }
3722
3723 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3724 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3725 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3726 return true;
3727 }
3728 }
3729
3730 // Handle ADD with large immediates.
3731 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3732 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3733 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3734
3735 // Handle immediates in the range [-4096,-2049] or [2017, 4063]. We can save
3736 // one instruction by folding adjustment (-2048 or 2016) into the address.
3737 // The upper bound keeps CVal - 2016 within simm12 ([−2048, 2047]).
3738 if ((-2049 >= CVal && CVal >= -4096) || (4063 >= CVal && CVal >= 2017)) {
3739 int64_t Adj = CVal < 0 ? -2048 : 2016;
3740 int64_t AdjustedOffset = CVal - Adj;
3741 Base =
3742 SDValue(CurDAG->getMachineNode(
3743 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3744 Op2: CurDAG->getSignedTargetConstant(Val: AdjustedOffset, DL, VT)),
3745 0);
3746 Offset = CurDAG->getSignedTargetConstant(Val: Adj, DL, VT);
3747 return true;
3748 }
3749
3750 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3751 Offset, /*IsPrefetch=*/true)) {
3752 // Insert an ADD instruction with the materialized Hi52 bits.
3753 Base = SDValue(
3754 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3755 0);
3756 return true;
3757 }
3758 }
3759
3760 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3761 /*IsPrefetch=*/true))
3762 return true;
3763
3764 Base = Addr;
3765 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3766 return true;
3767}
3768
3769/// Return true if this a load/store that we have a RegRegScale instruction for.
3770static bool isRegRegScaleLoadOrStore(SDNode *User, SDValue Add,
3771 const RISCVSubtarget &Subtarget) {
3772 unsigned UserOpc = User->getOpcode();
3773 if (UserOpc != ISD::LOAD && UserOpc != ISD::STORE)
3774 return false;
3775 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3776 // Zilx only provides indexed loads, so it must not enable reg+reg-scale
3777 // address folding for stores. XTheadMemIdx and Xqcisls have scaled stores.
3778 bool HasScalarIntegerMemIdx =
3779 Subtarget.hasVendorXTHeadMemIdx() || Subtarget.hasVendorXqcisls() ||
3780 (Subtarget.hasStdExtZilx() && UserOpc == ISD::LOAD);
3781 if (!(VT.isScalarInteger() && HasScalarIntegerMemIdx) &&
3782 !((VT == MVT::f32 || VT == MVT::f64) &&
3783 Subtarget.hasVendorXTHeadFMemIdx()))
3784 return false;
3785 // Don't allow stores of the value. It must be used as the address.
3786 if (UserOpc == ISD::STORE && cast<StoreSDNode>(Val: User)->getValue() == Add)
3787 return false;
3788
3789 return true;
3790}
3791
3792/// Is it profitable to fold this Add into RegRegScale load/store. If \p
3793/// Shift is non-null, then we have matched a shl+add. We allow reassociating
3794/// (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2)) if there is a
3795/// single addi and we don't have a SHXADD instruction we could use.
3796/// FIXME: May still need to check how many and what kind of users the SHL has.
3797static bool isWorthFoldingIntoRegRegScale(const RISCVSubtarget &Subtarget,
3798 SDValue Add,
3799 SDValue Shift = SDValue()) {
3800 bool FoundADDI = false;
3801 for (auto *User : Add->users()) {
3802 if (isRegRegScaleLoadOrStore(User, Add, Subtarget))
3803 continue;
3804
3805 // Allow a single ADDI that is used by loads/stores if we matched a shift.
3806 if (!Shift || FoundADDI || User->getOpcode() != ISD::ADD ||
3807 !isa<ConstantSDNode>(Val: User->getOperand(Num: 1)) ||
3808 !isInt<12>(x: cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue()))
3809 return false;
3810
3811 FoundADDI = true;
3812
3813 // If we have a SHXADD instruction, prefer that over reassociating an ADDI.
3814 assert(Shift.getOpcode() == ISD::SHL);
3815 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
3816 if (Subtarget.hasShlAdd(ShAmt: ShiftAmt))
3817 return false;
3818
3819 // All users of the ADDI should be load/store.
3820 for (auto *ADDIUser : User->users())
3821 if (!isRegRegScaleLoadOrStore(User: ADDIUser, Add: SDValue(User, 0), Subtarget))
3822 return false;
3823 }
3824
3825 return true;
3826}
3827
3828bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
3829 ArrayRef<unsigned> Amounts,
3830 SDValue &Base, SDValue &Index,
3831 SDValue &Scale) {
3832 if (Addr.getOpcode() != ISD::ADD)
3833 return false;
3834 SDValue LHS = Addr.getOperand(i: 0);
3835 SDValue RHS = Addr.getOperand(i: 1);
3836
3837 EVT VT = Addr.getSimpleValueType();
3838 auto SelectShl = [this, VT, Amounts](SDValue N, SDValue &Index,
3839 SDValue &Shift) {
3840 if (N.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
3841 return false;
3842
3843 // Only match shifts by a value in range [0, MaxShiftAmount].
3844 unsigned ShiftAmt = N.getConstantOperandVal(i: 1);
3845 if (!llvm::is_contained(Range: Amounts, Element: ShiftAmt))
3846 return false;
3847
3848 Index = N.getOperand(i: 0);
3849 Shift = CurDAG->getTargetConstant(Val: ShiftAmt, DL: SDLoc(N), VT);
3850 return true;
3851 };
3852
3853 if (auto *C1 = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3854 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
3855 if (LHS.getOpcode() == ISD::ADD &&
3856 !isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
3857 isInt<12>(x: C1->getSExtValue())) {
3858 if (SelectShl(LHS.getOperand(i: 1), Index, Scale) &&
3859 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 1))) {
3860 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
3861 DL: SDLoc(Addr), VT);
3862 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
3863 Op1: LHS.getOperand(i: 0), Op2: C1Val),
3864 0);
3865 return true;
3866 }
3867
3868 // Add is commutative so we need to check both operands.
3869 if (SelectShl(LHS.getOperand(i: 0), Index, Scale) &&
3870 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 0))) {
3871 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
3872 DL: SDLoc(Addr), VT);
3873 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
3874 Op1: LHS.getOperand(i: 1), Op2: C1Val),
3875 0);
3876 return true;
3877 }
3878 }
3879
3880 // Don't match add with constants.
3881 // FIXME: Is this profitable for large constants that have 0s in the lower
3882 // 12 bits that we can materialize with LUI?
3883 return false;
3884 }
3885
3886 // Try to match a shift on the RHS.
3887 if (SelectShl(RHS, Index, Scale)) {
3888 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: RHS))
3889 return false;
3890 Base = LHS;
3891 return true;
3892 }
3893
3894 // Try to match a shift on the LHS.
3895 if (SelectShl(LHS, Index, Scale)) {
3896 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: LHS))
3897 return false;
3898 Base = RHS;
3899 return true;
3900 }
3901
3902 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr))
3903 return false;
3904
3905 // Bail out if 0 is not in candidate shift amounts.
3906 if (!llvm::is_contained(Range&: Amounts, Element: 0))
3907 return false;
3908
3909 Base = LHS;
3910 Index = RHS;
3911 Scale = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT);
3912 return true;
3913}
3914
3915bool RISCVDAGToDAGISel::SelectAddrRegZextRegScale(SDValue Addr,
3916 ArrayRef<unsigned> Amounts,
3917 unsigned Bits, SDValue &Base,
3918 SDValue &Index,
3919 SDValue &Scale) {
3920 if (!SelectAddrRegRegScale(Addr, Amounts, Base, Index, Scale))
3921 return false;
3922
3923 if (Index.getOpcode() == ISD::AND) {
3924 auto *C = dyn_cast<ConstantSDNode>(Val: Index.getOperand(i: 1));
3925 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
3926 Index = Index.getOperand(i: 0);
3927 return true;
3928 }
3929 }
3930
3931 return false;
3932}
3933
3934bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base,
3935 SDValue &Offset) {
3936 if (Addr.getOpcode() != ISD::ADD)
3937 return false;
3938
3939 if (isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)))
3940 return false;
3941
3942 Base = Addr.getOperand(i: 0);
3943 Offset = Addr.getOperand(i: 1);
3944 return true;
3945}
3946
3947bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
3948 SDValue &ShAmt) {
3949 ShAmt = N;
3950
3951 // Peek through zext.
3952 if (ShAmt->getOpcode() == ISD::ZERO_EXTEND)
3953 ShAmt = ShAmt.getOperand(i: 0);
3954
3955 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
3956 // amount. If there is an AND on the shift amount, we can bypass it if it
3957 // doesn't affect any of those bits.
3958 if (ShAmt.getOpcode() == ISD::AND &&
3959 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
3960 const APInt &AndMask = ShAmt.getConstantOperandAPInt(i: 1);
3961
3962 // Since the max shift amount is a power of 2 we can subtract 1 to make a
3963 // mask that covers the bits needed to represent all shift amounts.
3964 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
3965 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
3966
3967 if (ShMask.isSubsetOf(RHS: AndMask)) {
3968 ShAmt = ShAmt.getOperand(i: 0);
3969 } else {
3970 // SimplifyDemandedBits may have optimized the mask so try restoring any
3971 // bits that are known zero.
3972 KnownBits Known = CurDAG->computeKnownBits(Op: ShAmt.getOperand(i: 0));
3973 if (!ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
3974 return true;
3975 ShAmt = ShAmt.getOperand(i: 0);
3976 }
3977 }
3978
3979 if (ShAmt.getOpcode() == ISD::ADD &&
3980 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
3981 uint64_t Imm = ShAmt.getConstantOperandVal(i: 1);
3982 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
3983 // to avoid the ADD.
3984 if (Imm != 0 && Imm % ShiftWidth == 0) {
3985 ShAmt = ShAmt.getOperand(i: 0);
3986 return true;
3987 }
3988 } else if (ShAmt.getOpcode() == ISD::SUB &&
3989 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 0))) {
3990 uint64_t Imm = ShAmt.getConstantOperandVal(i: 0);
3991 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
3992 // generate a NEG instead of a SUB of a constant.
3993 if (Imm != 0 && Imm % ShiftWidth == 0) {
3994 SDLoc DL(ShAmt);
3995 EVT VT = ShAmt.getValueType();
3996 SDValue Zero = CurDAG->getRegister(Reg: RISCV::X0, VT);
3997 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
3998 MachineSDNode *Neg = CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero,
3999 Op2: ShAmt.getOperand(i: 1));
4000 ShAmt = SDValue(Neg, 0);
4001 return true;
4002 }
4003 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4004 // to generate a NOT instead of a SUB of a constant.
4005 if (Imm % ShiftWidth == ShiftWidth - 1) {
4006 SDLoc DL(ShAmt);
4007 EVT VT = ShAmt.getValueType();
4008 MachineSDNode *Not = CurDAG->getMachineNode(
4009 Opcode: RISCV::XORI, dl: DL, VT, Op1: ShAmt.getOperand(i: 1),
4010 Op2: CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/IsTarget: true));
4011 ShAmt = SDValue(Not, 0);
4012 return true;
4013 }
4014 }
4015
4016 return true;
4017}
4018
4019/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
4020/// check for equality with 0. This function emits instructions that convert the
4021/// seteq/setne into something that can be compared with 0.
4022/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
4023/// ISD::SETNE).
4024bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
4025 SDValue &Val, bool OneUse) {
4026 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
4027 "Unexpected condition code!");
4028
4029 // We're looking for a setcc.
4030 if (N->getOpcode() != ISD::SETCC)
4031 return false;
4032
4033 if (OneUse && !N->hasOneUse())
4034 return false;
4035
4036 // Must be an equality comparison.
4037 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
4038 if (CCVal != ExpectedCCVal)
4039 return false;
4040
4041 SDValue LHS = N->getOperand(Num: 0);
4042 SDValue RHS = N->getOperand(Num: 1);
4043
4044 if (!LHS.getValueType().isScalarInteger())
4045 return false;
4046
4047 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
4048 if (isNullConstant(V: RHS)) {
4049 Val = LHS;
4050 return true;
4051 }
4052
4053 SDLoc DL(N);
4054
4055 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4056 int64_t CVal = C->getSExtValue();
4057 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
4058 // non-zero otherwise.
4059 if (CVal == -2048) {
4060 Val = SDValue(
4061 CurDAG->getMachineNode(
4062 Opcode: RISCV::XORI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4063 Op2: CurDAG->getSignedTargetConstant(Val: CVal, DL, VT: N->getValueType(ResNo: 0))),
4064 0);
4065 return true;
4066 }
4067 // If the RHS is [-2047,2048], we can use addi/addiw with -RHS to produce 0
4068 // if the LHS is equal to the RHS and non-zero otherwise.
4069 if (isInt<12>(x: CVal) || CVal == 2048) {
4070 unsigned Opc = RISCV::ADDI;
4071 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4072 cast<VTSDNode>(Val: LHS.getOperand(i: 1))->getVT() == MVT::i32) {
4073 Opc = RISCV::ADDIW;
4074 LHS = LHS.getOperand(i: 0);
4075 }
4076
4077 Val = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4078 Op2: CurDAG->getSignedTargetConstant(
4079 Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4080 0);
4081 return true;
4082 }
4083 if (isPowerOf2_64(Value: CVal) && Subtarget->hasStdExtZbs()) {
4084 Val = SDValue(
4085 CurDAG->getMachineNode(
4086 Opcode: RISCV::BINVI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4087 Op2: CurDAG->getTargetConstant(Val: Log2_64(Value: CVal), DL, VT: N->getValueType(ResNo: 0))),
4088 0);
4089 return true;
4090 }
4091 // Same as the addi case above but for larger immediates (signed 26-bit) use
4092 // the QC_E_ADDI instruction from the Xqcilia extension, if available. Avoid
4093 // anything which can be done with a single lui as it might be compressible.
4094 if (Subtarget->hasVendorXqcilia() && isInt<26>(x: CVal) &&
4095 (CVal & 0xFFF) != 0) {
4096 Val = SDValue(
4097 CurDAG->getMachineNode(
4098 Opcode: RISCV::QC_E_ADDI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4099 Op2: CurDAG->getSignedTargetConstant(Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4100 0);
4101 return true;
4102 }
4103 }
4104
4105 // If nothing else we can XOR the LHS and RHS to produce zero if they are
4106 // equal and a non-zero value if they aren't.
4107 Val = SDValue(
4108 CurDAG->getMachineNode(Opcode: RISCV::XOR, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS, Op2: RHS), 0);
4109 return true;
4110}
4111
4112bool RISCVDAGToDAGISel::selectSExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4113 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4114 cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT().getSizeInBits() == Bits) {
4115 Val = N.getOperand(i: 0);
4116 return true;
4117 }
4118
4119 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
4120 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
4121 return N;
4122
4123 SDValue N0 = N.getOperand(i: 0);
4124 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4125 N.getConstantOperandVal(i: 1) == ShiftAmt &&
4126 N0.getConstantOperandVal(i: 1) == ShiftAmt)
4127 return N0.getOperand(i: 0);
4128
4129 return N;
4130 };
4131
4132 MVT VT = N.getSimpleValueType();
4133 if (CurDAG->ComputeNumSignBits(Op: N) > (VT.getSizeInBits() - Bits)) {
4134 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
4135 return true;
4136 }
4137
4138 return false;
4139}
4140
4141bool RISCVDAGToDAGISel::selectZExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4142 if (N.getOpcode() == ISD::AND) {
4143 auto *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
4144 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4145 Val = N.getOperand(i: 0);
4146 return true;
4147 }
4148 }
4149 MVT VT = N.getSimpleValueType();
4150 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: Bits);
4151 if (CurDAG->MaskedValueIsZero(Op: N, Mask)) {
4152 Val = N;
4153 return true;
4154 }
4155
4156 return false;
4157}
4158
4159/// Look for various patterns that can be done with a SHL that can be folded
4160/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
4161/// SHXADD we are trying to match.
4162bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
4163 SDValue &Val) {
4164 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4165 SDValue N0 = N.getOperand(i: 0);
4166
4167 if (bool LeftShift = N0.getOpcode() == ISD::SHL;
4168 (LeftShift || N0.getOpcode() == ISD::SRL) &&
4169 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4170 uint64_t Mask = N.getConstantOperandVal(i: 1);
4171 unsigned C2 = N0.getConstantOperandVal(i: 1);
4172
4173 unsigned XLen = Subtarget->getXLen();
4174 if (LeftShift)
4175 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4176 else
4177 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
4178
4179 if (isShiftedMask_64(Value: Mask)) {
4180 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4181 unsigned Trailing = llvm::countr_zero(Val: Mask);
4182 if (Trailing != ShAmt)
4183 return false;
4184
4185 unsigned Opcode;
4186 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
4187 // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
4188 // followed by a SHXADD with c3 for the X amount.
4189 if (LeftShift && Leading == 0 && C2 < Trailing)
4190 Opcode = RISCV::SRLI;
4191 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
4192 // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
4193 // followed by a SHXADD with c3 for the X amount.
4194 else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
4195 Opcode = RISCV::SRLIW;
4196 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
4197 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
4198 // followed by a SHXADD using c3 for the X amount.
4199 else if (!LeftShift && Leading == C2)
4200 Opcode = RISCV::SRLI;
4201 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
4202 // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
4203 // followed by a SHXADD using c3 for the X amount.
4204 else if (!LeftShift && Leading == 32 + C2)
4205 Opcode = RISCV::SRLIW;
4206 else
4207 return false;
4208
4209 SDLoc DL(N);
4210 EVT VT = N.getValueType();
4211 ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
4212 Val = SDValue(
4213 CurDAG->getMachineNode(Opcode, dl: DL, VT, Op1: N0.getOperand(i: 0),
4214 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT)),
4215 0);
4216 return true;
4217 }
4218 } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
4219 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4220 uint64_t Mask = N.getConstantOperandVal(i: 1);
4221 unsigned C2 = N0.getConstantOperandVal(i: 1);
4222
4223 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
4224 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
4225 // use (srli (srai y, c2 - c3), c3 + c4) followed by a SHXADD with c4 as
4226 // the X amount.
4227 if (isShiftedMask_64(Value: Mask)) {
4228 unsigned XLen = Subtarget->getXLen();
4229 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4230 unsigned Trailing = llvm::countr_zero(Val: Mask);
4231 if (C2 > Leading && Leading > 0 && Trailing == ShAmt) {
4232 SDLoc DL(N);
4233 EVT VT = N.getValueType();
4234 Val = SDValue(CurDAG->getMachineNode(
4235 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4236 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT)),
4237 0);
4238 Val = SDValue(CurDAG->getMachineNode(
4239 Opcode: RISCV::SRLI, dl: DL, VT, Op1: Val,
4240 Op2: CurDAG->getTargetConstant(Val: Leading + ShAmt, DL, VT)),
4241 0);
4242 return true;
4243 }
4244 }
4245 }
4246 } else if (bool LeftShift = N.getOpcode() == ISD::SHL;
4247 (LeftShift || N.getOpcode() == ISD::SRL) &&
4248 isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4249 SDValue N0 = N.getOperand(i: 0);
4250 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
4251 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4252 uint64_t Mask = N0.getConstantOperandVal(i: 1);
4253 if (isShiftedMask_64(Value: Mask)) {
4254 unsigned C1 = N.getConstantOperandVal(i: 1);
4255 unsigned XLen = Subtarget->getXLen();
4256 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4257 unsigned Trailing = llvm::countr_zero(Val: Mask);
4258 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
4259 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
4260 if (LeftShift && Leading == 32 && Trailing > 0 &&
4261 (Trailing + C1) == ShAmt) {
4262 SDLoc DL(N);
4263 EVT VT = N.getValueType();
4264 Val = SDValue(CurDAG->getMachineNode(
4265 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4266 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4267 0);
4268 return true;
4269 }
4270 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
4271 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
4272 if (!LeftShift && Leading == 32 && Trailing > C1 &&
4273 (Trailing - C1) == ShAmt) {
4274 SDLoc DL(N);
4275 EVT VT = N.getValueType();
4276 Val = SDValue(CurDAG->getMachineNode(
4277 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4278 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4279 0);
4280 return true;
4281 }
4282 }
4283 }
4284 }
4285
4286 return false;
4287}
4288
4289/// Look for various patterns that can be done with a SHL that can be folded
4290/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
4291/// SHXADD_UW we are trying to match.
4292bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
4293 SDValue &Val) {
4294 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
4295 N.hasOneUse()) {
4296 SDValue N0 = N.getOperand(i: 0);
4297 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4298 N0.hasOneUse()) {
4299 uint64_t Mask = N.getConstantOperandVal(i: 1);
4300 unsigned C2 = N0.getConstantOperandVal(i: 1);
4301
4302 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4303
4304 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
4305 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
4306 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
4307 if (isShiftedMask_64(Value: Mask)) {
4308 unsigned Leading = llvm::countl_zero(Val: Mask);
4309 unsigned Trailing = llvm::countr_zero(Val: Mask);
4310 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
4311 SDLoc DL(N);
4312 EVT VT = N.getValueType();
4313 Val = SDValue(CurDAG->getMachineNode(
4314 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4315 Op2: CurDAG->getTargetConstant(Val: C2 - ShAmt, DL, VT)),
4316 0);
4317 return true;
4318 }
4319 }
4320 }
4321 }
4322
4323 return false;
4324}
4325
4326bool RISCVDAGToDAGISel::orDisjoint(const SDNode *N) const {
4327 assert(N->getOpcode() == ISD::OR || N->getOpcode() == RISCVISD::OR_VL);
4328 if (N->getFlags().hasDisjoint())
4329 return true;
4330 return CurDAG->haveNoCommonBitsSet(A: N->getOperand(Num: 0), B: N->getOperand(Num: 1));
4331}
4332
4333bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
4334 SDValue N, SDValue &Val) {
4335 int OrigCost = RISCVMatInt::getIntMatCost(Val: APInt(64, OrigImm), Size: 64, STI: *Subtarget,
4336 /*CompressionCost=*/true);
4337 int Cost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
4338 /*CompressionCost=*/true);
4339 if (OrigCost <= Cost)
4340 return false;
4341
4342 Val = selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm, Subtarget: *Subtarget);
4343 return true;
4344}
4345
4346bool RISCVDAGToDAGISel::selectZExtImm32(SDValue N, SDValue &Val) {
4347 if (!isa<ConstantSDNode>(Val: N))
4348 return false;
4349 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4350 if ((Imm >> 31) != 1)
4351 return false;
4352
4353 for (const SDNode *U : N->users()) {
4354 switch (U->getOpcode()) {
4355 case ISD::ADD:
4356 break;
4357 case ISD::OR:
4358 if (orDisjoint(N: U))
4359 break;
4360 return false;
4361 default:
4362 return false;
4363 }
4364 }
4365
4366 return selectImm64IfCheaper(Imm: 0xffffffff00000000 | Imm, OrigImm: Imm, N, Val);
4367}
4368
4369bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
4370 if (!isa<ConstantSDNode>(Val: N))
4371 return false;
4372 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4373 if (isInt<32>(x: Imm))
4374 return false;
4375 if (Imm == INT64_MIN)
4376 return false;
4377
4378 for (const SDNode *U : N->users()) {
4379 switch (U->getOpcode()) {
4380 case ISD::ADD:
4381 break;
4382 case RISCVISD::VMV_V_X_VL:
4383 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4384 return V->getOpcode() == ISD::ADD ||
4385 V->getOpcode() == RISCVISD::ADD_VL;
4386 }))
4387 return false;
4388 break;
4389 default:
4390 return false;
4391 }
4392 }
4393
4394 return selectImm64IfCheaper(Imm: -Imm, OrigImm: Imm, N, Val);
4395}
4396
4397bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
4398 if (!isa<ConstantSDNode>(Val: N))
4399 return false;
4400 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4401
4402 // For 32-bit signed constants, we can only substitute LUI+ADDI with LUI.
4403 if (isInt<32>(x: Imm) && ((Imm & 0xfff) != 0xfff || Imm == -1))
4404 return false;
4405
4406 // Abandon this transform if the constant is needed elsewhere.
4407 for (const SDNode *U : N->users()) {
4408 switch (U->getOpcode()) {
4409 case ISD::AND:
4410 case ISD::OR:
4411 case ISD::XOR:
4412 if (!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()))
4413 return false;
4414 break;
4415 case RISCVISD::VMV_V_X_VL:
4416 if (!Subtarget->hasStdExtZvkb())
4417 return false;
4418 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4419 return V->getOpcode() == ISD::AND ||
4420 V->getOpcode() == RISCVISD::AND_VL;
4421 }))
4422 return false;
4423 break;
4424 default:
4425 return false;
4426 }
4427 }
4428
4429 if (isInt<32>(x: Imm)) {
4430 Val =
4431 selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm: ~Imm, Subtarget: *Subtarget);
4432 return true;
4433 }
4434
4435 // For 64-bit constants, the instruction sequences get complex,
4436 // so we select inverted only if it's cheaper.
4437 return selectImm64IfCheaper(Imm: ~Imm, OrigImm: Imm, N, Val);
4438}
4439
4440static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
4441 unsigned Bits,
4442 const TargetInstrInfo *TII) {
4443 unsigned MCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: User->getMachineOpcode());
4444
4445 if (!MCOpcode)
4446 return false;
4447
4448 const MCInstrDesc &MCID = TII->get(Opcode: User->getMachineOpcode());
4449 const uint64_t TSFlags = MCID.TSFlags;
4450 if (!RISCVII::hasSEWOp(TSFlags))
4451 return false;
4452 assert(RISCVII::hasVLOp(TSFlags));
4453
4454 unsigned ChainOpIdx = User->getNumOperands() - 1;
4455 bool HasChainOp = User->getOperand(Num: ChainOpIdx).getValueType() == MVT::Other;
4456 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
4457 unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
4458 const unsigned Log2SEW = User->getConstantOperandVal(Num: VLIdx + 1);
4459
4460 if (UserOpNo == VLIdx)
4461 return false;
4462
4463 auto NumDemandedBits =
4464 RISCV::getVectorLowDemandedScalarBits(Opcode: MCOpcode, Log2SEW);
4465 return NumDemandedBits && Bits >= *NumDemandedBits;
4466}
4467
4468// Return true if all users of this SDNode* only consume the lower \p Bits.
4469// This can be used to form W instructions for add/sub/mul/shl even when the
4470// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
4471// SimplifyDemandedBits has made it so some users see a sext_inreg and some
4472// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
4473// the add/sub/mul/shl to become non-W instructions. By checking the users we
4474// may be able to use a W instruction and CSE with the other instruction if
4475// this has happened. We could try to detect that the CSE opportunity exists
4476// before doing this, but that would be more complicated.
4477bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits,
4478 const unsigned Depth) const {
4479 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
4480 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
4481 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
4482 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
4483 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
4484 isa<ConstantSDNode>(Node) || Depth != 0) &&
4485 "Unexpected opcode");
4486
4487 if (Depth >= SelectionDAG::MaxRecursionDepth)
4488 return false;
4489
4490 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
4491 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
4492 if (Depth == 0 && !Node->getValueType(ResNo: 0).isScalarInteger())
4493 return false;
4494
4495 for (SDUse &Use : Node->uses()) {
4496 SDNode *User = Use.getUser();
4497 // Users of this node should have already been instruction selected
4498 if (!User->isMachineOpcode())
4499 return false;
4500
4501 // TODO: Add more opcodes?
4502 switch (User->getMachineOpcode()) {
4503 default:
4504 if (vectorPseudoHasAllNBitUsers(User, UserOpNo: Use.getOperandNo(), Bits, TII))
4505 break;
4506 return false;
4507 case RISCV::ADDW:
4508 case RISCV::ADDIW:
4509 case RISCV::SUBW:
4510 case RISCV::MULW:
4511 case RISCV::SLLW:
4512 case RISCV::SLLIW:
4513 case RISCV::SRAW:
4514 case RISCV::SRAIW:
4515 case RISCV::SRLW:
4516 case RISCV::SRLIW:
4517 case RISCV::DIVW:
4518 case RISCV::DIVUW:
4519 case RISCV::REMW:
4520 case RISCV::REMUW:
4521 case RISCV::ROLW:
4522 case RISCV::RORW:
4523 case RISCV::RORIW:
4524 case RISCV::CLSW:
4525 case RISCV::CLZW:
4526 case RISCV::CTZW:
4527 case RISCV::CPOPW:
4528 case RISCV::SLLI_UW:
4529 case RISCV::ABSW:
4530 case RISCV::FMV_W_X:
4531 case RISCV::FCVT_H_W:
4532 case RISCV::FCVT_H_W_INX:
4533 case RISCV::FCVT_H_WU:
4534 case RISCV::FCVT_H_WU_INX:
4535 case RISCV::FCVT_S_W:
4536 case RISCV::FCVT_S_W_INX:
4537 case RISCV::FCVT_S_WU:
4538 case RISCV::FCVT_S_WU_INX:
4539 case RISCV::FCVT_D_W:
4540 case RISCV::FCVT_D_W_INX:
4541 case RISCV::FCVT_D_WU:
4542 case RISCV::FCVT_D_WU_INX:
4543 case RISCV::TH_REVW:
4544 case RISCV::TH_SRRIW:
4545 if (Bits >= 32)
4546 break;
4547 return false;
4548 case RISCV::SLL:
4549 case RISCV::SRA:
4550 case RISCV::SRL:
4551 case RISCV::ROL:
4552 case RISCV::ROR:
4553 case RISCV::BSET:
4554 case RISCV::BCLR:
4555 case RISCV::BINV:
4556 // Shift amount operands only use log2(Xlen) bits.
4557 if (Use.getOperandNo() == 1 && Bits >= Log2_32(Value: Subtarget->getXLen()))
4558 break;
4559 return false;
4560 case RISCV::SLLI:
4561 // SLLI only uses the lower (XLen - ShAmt) bits.
4562 if (Bits >= Subtarget->getXLen() - User->getConstantOperandVal(Num: 1))
4563 break;
4564 return false;
4565 case RISCV::ANDI:
4566 if (Bits >= (unsigned)llvm::bit_width(Value: User->getConstantOperandVal(Num: 1)))
4567 break;
4568 goto RecCheck;
4569 case RISCV::ORI: {
4570 uint64_t Imm = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
4571 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(Value: ~Imm))
4572 break;
4573 [[fallthrough]];
4574 }
4575 case RISCV::AND:
4576 case RISCV::OR:
4577 case RISCV::XOR:
4578 case RISCV::XORI:
4579 case RISCV::ANDN:
4580 case RISCV::ORN:
4581 case RISCV::XNOR:
4582 case RISCV::SH1ADD:
4583 case RISCV::SH2ADD:
4584 case RISCV::SH3ADD:
4585 RecCheck:
4586 if (hasAllNBitUsers(Node: User, Bits, Depth: Depth + 1))
4587 break;
4588 return false;
4589 case RISCV::SRLI: {
4590 unsigned ShAmt = User->getConstantOperandVal(Num: 1);
4591 // If we are shifting right by less than Bits, and users don't demand any
4592 // bits that were shifted into [Bits-1:0], then we can consider this as an
4593 // N-Bit user.
4594 if (Bits > ShAmt && hasAllNBitUsers(Node: User, Bits: Bits - ShAmt, Depth: Depth + 1))
4595 break;
4596 return false;
4597 }
4598 case RISCV::SEXT_B:
4599 case RISCV::PACKH:
4600 if (Bits >= 8)
4601 break;
4602 return false;
4603 case RISCV::SEXT_H:
4604 case RISCV::FMV_H_X:
4605 case RISCV::ZEXT_H_RV32:
4606 case RISCV::ZEXT_H_RV64:
4607 case RISCV::PACKW:
4608 if (Bits >= 16)
4609 break;
4610 return false;
4611 case RISCV::PACK:
4612 if (Bits >= (Subtarget->getXLen() / 2))
4613 break;
4614 return false;
4615 case RISCV::PPAIRE_H:
4616 // If only the lower 32-bits of the result are used, then only the
4617 // lower 16 bits of the inputs are used.
4618 if (Bits >= 16 && hasAllNBitUsers(Node: User, Bits: 32, Depth: Depth + 1))
4619 break;
4620 return false;
4621 case RISCV::ADD_UW:
4622 case RISCV::SH1ADD_UW:
4623 case RISCV::SH2ADD_UW:
4624 case RISCV::SH3ADD_UW:
4625 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
4626 // 32 bits.
4627 if (Use.getOperandNo() == 0 && Bits >= 32)
4628 break;
4629 return false;
4630 case RISCV::SB:
4631 if (Use.getOperandNo() == 0 && Bits >= 8)
4632 break;
4633 return false;
4634 case RISCV::SH:
4635 if (Use.getOperandNo() == 0 && Bits >= 16)
4636 break;
4637 return false;
4638 case RISCV::SW:
4639 if (Use.getOperandNo() == 0 && Bits >= 32)
4640 break;
4641 return false;
4642 case RISCV::TH_EXT:
4643 case RISCV::TH_EXTU: {
4644 unsigned Msb = User->getConstantOperandVal(Num: 1);
4645 unsigned Lsb = User->getConstantOperandVal(Num: 2);
4646 // Behavior of Msb < Lsb is not well documented.
4647 if (Msb >= Lsb && Bits > Msb)
4648 break;
4649 return false;
4650 }
4651 }
4652 }
4653
4654 return true;
4655}
4656
4657// Select a constant that can be represented as (sign_extend(imm5) << imm2).
4658bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
4659 SDValue &Shl2) {
4660 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4661 if (!C)
4662 return false;
4663
4664 int64_t Offset = C->getSExtValue();
4665 for (unsigned Shift = 0; Shift < 4; Shift++) {
4666 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0)) {
4667 EVT VT = N->getValueType(ResNo: 0);
4668 Simm5 = CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(N), VT);
4669 Shl2 = CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(N), VT);
4670 return true;
4671 }
4672 }
4673
4674 return false;
4675}
4676
4677// Select VL as a 5 bit immediate or a value that will become a register. This
4678// allows us to choose between VSETIVLI or VSETVLI later.
4679bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
4680 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4681 if (C && isUInt<5>(x: C->getZExtValue())) {
4682 VL = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(N),
4683 VT: N->getValueType(ResNo: 0));
4684 } else if (C && C->isAllOnes()) {
4685 // Treat all ones as VLMax.
4686 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4687 VT: N->getValueType(ResNo: 0));
4688 } else if (isa<RegisterSDNode>(Val: N) &&
4689 cast<RegisterSDNode>(Val&: N)->getReg() == RISCV::X0) {
4690 // All our VL operands use an operand that allows GPRNoX0 or an immediate
4691 // as the register class. Convert X0 to a special immediate to pass the
4692 // MachineVerifier. This is recognized specially by the vsetvli insertion
4693 // pass.
4694 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4695 VT: N->getValueType(ResNo: 0));
4696 } else {
4697 VL = N;
4698 }
4699
4700 return true;
4701}
4702
4703static SDValue findVSplat(SDValue N) {
4704 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
4705 if (!N.getOperand(i: 0).isUndef())
4706 return SDValue();
4707 N = N.getOperand(i: 1);
4708 }
4709 SDValue Splat = N;
4710 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
4711 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
4712 !Splat.getOperand(i: 0).isUndef())
4713 return SDValue();
4714 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
4715 return Splat;
4716}
4717
4718bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
4719 SDValue Splat = findVSplat(N);
4720 if (!Splat)
4721 return false;
4722
4723 SplatVal = Splat.getOperand(i: 1);
4724 return true;
4725}
4726
4727static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
4728 SelectionDAG &DAG,
4729 const RISCVSubtarget &Subtarget,
4730 std::function<bool(int64_t)> ValidateImm,
4731 bool Decrement = false) {
4732 SDValue Splat = findVSplat(N);
4733 if (!Splat || !isa<ConstantSDNode>(Val: Splat.getOperand(i: 1)))
4734 return false;
4735
4736 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
4737 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
4738 "Unexpected splat operand type");
4739
4740 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
4741 // type is wider than the resulting vector element type: an implicit
4742 // truncation first takes place. Therefore, perform a manual
4743 // truncation/sign-extension in order to ignore any truncated bits and catch
4744 // any zero-extended immediate.
4745 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
4746 // sign-extending to (XLenVT -1).
4747 APInt SplatConst = Splat.getConstantOperandAPInt(i: 1).sextOrTrunc(width: SplatEltSize);
4748
4749 int64_t SplatImm = SplatConst.getSExtValue();
4750
4751 if (!ValidateImm(SplatImm))
4752 return false;
4753
4754 if (Decrement)
4755 SplatImm -= 1;
4756
4757 SplatVal =
4758 DAG.getSignedTargetConstant(Val: SplatImm, DL: SDLoc(N), VT: Subtarget.getXLenVT());
4759 return true;
4760}
4761
4762bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
4763 return selectVSplatImmHelper(N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4764 ValidateImm: [](int64_t Imm) { return isInt<5>(x: Imm); });
4765}
4766
4767bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
4768 return selectVSplatImmHelper(
4769 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4770 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4771 /*Decrement=*/true);
4772}
4773
4774bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal) {
4775 return selectVSplatImmHelper(
4776 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4777 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4778 /*Decrement=*/false);
4779}
4780
4781bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
4782 SDValue &SplatVal) {
4783 return selectVSplatImmHelper(
4784 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4785 ValidateImm: [](int64_t Imm) { return Imm != 0 && Imm >= -15 && Imm <= 16; },
4786 /*Decrement=*/true);
4787}
4788
4789bool RISCVDAGToDAGISel::selectVSplatUimm(SDValue N, unsigned Bits,
4790 SDValue &SplatVal) {
4791 return selectVSplatImmHelper(
4792 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4793 ValidateImm: [Bits](int64_t Imm) { return isUIntN(N: Bits, x: Imm); });
4794}
4795
4796bool RISCVDAGToDAGISel::selectVSplatImm64Neg(SDValue N, SDValue &SplatVal) {
4797 SDValue Splat = findVSplat(N);
4798 return Splat && selectNegImm(N: Splat.getOperand(i: 1), Val&: SplatVal);
4799}
4800
4801bool RISCVDAGToDAGISel::selectLow8BitsVSplat(SDValue N, SDValue &SplatVal) {
4802 auto IsExtOrTrunc = [](SDValue N) {
4803 switch (N->getOpcode()) {
4804 case ISD::SIGN_EXTEND:
4805 case ISD::ZERO_EXTEND:
4806 // There's no passthru on these _VL nodes so any VL/mask is ok, since any
4807 // inactive elements will be undef.
4808 case RISCVISD::TRUNCATE_VECTOR_VL:
4809 case RISCVISD::VSEXT_VL:
4810 case RISCVISD::VZEXT_VL:
4811 return true;
4812 default:
4813 return false;
4814 }
4815 };
4816
4817 // We can have multiple nested nodes, so unravel them all if needed.
4818 while (IsExtOrTrunc(N)) {
4819 if (!N.hasOneUse() || N.getScalarValueSizeInBits() < 8)
4820 return false;
4821 N = N->getOperand(Num: 0);
4822 }
4823
4824 return selectVSplat(N, SplatVal);
4825}
4826
4827bool RISCVDAGToDAGISel::selectScalarFPAsInt(SDValue N, SDValue &Imm) {
4828 // Allow bitcasts from XLenVT -> FP.
4829 if (N.getOpcode() == ISD::BITCAST &&
4830 N.getOperand(i: 0).getValueType() == Subtarget->getXLenVT()) {
4831 Imm = N.getOperand(i: 0);
4832 return true;
4833 }
4834 // Allow moves from XLenVT to FP.
4835 if (N.getOpcode() == RISCVISD::FMV_H_X ||
4836 N.getOpcode() == RISCVISD::FMV_W_X_RV64) {
4837 Imm = N.getOperand(i: 0);
4838 return true;
4839 }
4840
4841 // Otherwise, look for FP constants that can materialized with scalar int.
4842 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val: N.getNode());
4843 if (!CFP)
4844 return false;
4845 const APFloat &APF = CFP->getValueAPF();
4846 // td can handle +0.0 already.
4847 if (APF.isPosZero())
4848 return false;
4849
4850 MVT VT = CFP->getSimpleValueType(ResNo: 0);
4851
4852 MVT XLenVT = Subtarget->getXLenVT();
4853 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
4854 assert(APF.isNegZero() && "Unexpected constant.");
4855 return false;
4856 }
4857 SDLoc DL(N);
4858 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
4859 Subtarget: *Subtarget);
4860 return true;
4861}
4862
4863bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
4864 SDValue &Imm) {
4865 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N)) {
4866 int64_t ImmVal = SignExtend64(X: C->getSExtValue(), B: Width);
4867
4868 if (!isInt<5>(x: ImmVal))
4869 return false;
4870
4871 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N),
4872 VT: Subtarget->getXLenVT());
4873 return true;
4874 }
4875
4876 return false;
4877}
4878
4879// Match XOR with a VMSET_VL operand. Return the other operand.
4880bool RISCVDAGToDAGISel::selectVMNOTOp(SDValue N, SDValue &Res) {
4881 if (N.getOpcode() != ISD::XOR)
4882 return false;
4883
4884 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
4885 Res = N.getOperand(i: 1);
4886 return true;
4887 }
4888
4889 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
4890 Res = N.getOperand(i: 0);
4891 return true;
4892 }
4893
4894 return false;
4895}
4896
4897// Match VMXOR_VL with a VMSET_VL operand. Making sure that that VL operand
4898// matches the parent's VL. Return the other operand of the VMXOR_VL.
4899bool RISCVDAGToDAGISel::selectVMNOT_VLOp(SDNode *Parent, SDValue N,
4900 SDValue &Res) {
4901 if (N.getOpcode() != RISCVISD::VMXOR_VL)
4902 return false;
4903
4904 assert(Parent &&
4905 (Parent->getOpcode() == RISCVISD::VMAND_VL ||
4906 Parent->getOpcode() == RISCVISD::VMOR_VL ||
4907 Parent->getOpcode() == RISCVISD::VMXOR_VL) &&
4908 "Unexpected parent");
4909
4910 // The VL should match the parent.
4911 if (Parent->getOperand(Num: 2) != N->getOperand(Num: 2))
4912 return false;
4913
4914 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
4915 Res = N.getOperand(i: 1);
4916 return true;
4917 }
4918
4919 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
4920 Res = N.getOperand(i: 0);
4921 return true;
4922 }
4923
4924 return false;
4925}
4926
4927// Try to remove sext.w if the input is a W instruction or can be made into
4928// a W instruction cheaply.
4929bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
4930 // Look for the sext.w pattern, addiw rd, rs1, 0.
4931 if (N->getMachineOpcode() != RISCV::ADDIW ||
4932 !isNullConstant(V: N->getOperand(Num: 1)))
4933 return false;
4934
4935 SDValue N0 = N->getOperand(Num: 0);
4936 if (!N0.isMachineOpcode())
4937 return false;
4938
4939 switch (N0.getMachineOpcode()) {
4940 default:
4941 break;
4942 case RISCV::ADD:
4943 case RISCV::ADDI:
4944 case RISCV::SUB:
4945 case RISCV::MUL:
4946 case RISCV::SLLI: {
4947 // Convert sext.w+add/sub/mul to their W instructions. This will create
4948 // a new independent instruction. This improves latency.
4949 unsigned Opc;
4950 switch (N0.getMachineOpcode()) {
4951 default:
4952 llvm_unreachable("Unexpected opcode!");
4953 case RISCV::ADD: Opc = RISCV::ADDW; break;
4954 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
4955 case RISCV::SUB: Opc = RISCV::SUBW; break;
4956 case RISCV::MUL: Opc = RISCV::MULW; break;
4957 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
4958 }
4959
4960 SDValue N00 = N0.getOperand(i: 0);
4961 SDValue N01 = N0.getOperand(i: 1);
4962
4963 // Shift amount needs to be uimm5.
4964 if (N0.getMachineOpcode() == RISCV::SLLI &&
4965 !isUInt<5>(x: cast<ConstantSDNode>(Val&: N01)->getSExtValue()))
4966 break;
4967
4968 SDNode *Result =
4969 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VT: N->getValueType(ResNo: 0),
4970 Op1: N00, Op2: N01);
4971 ReplaceUses(F: N, T: Result);
4972 return true;
4973 }
4974 case RISCV::ADDW:
4975 case RISCV::ADDIW:
4976 case RISCV::SUBW:
4977 case RISCV::MULW:
4978 case RISCV::SLLIW:
4979 case RISCV::PACKW:
4980 case RISCV::TH_MULAW:
4981 case RISCV::TH_MULAH:
4982 case RISCV::TH_MULSW:
4983 case RISCV::TH_MULSH:
4984 if (N0.getValueType() == MVT::i32)
4985 break;
4986
4987 // Result is already sign extended just remove the sext.w.
4988 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
4989 ReplaceUses(F: N, T: N0.getNode());
4990 return true;
4991 }
4992
4993 return false;
4994}
4995
4996static bool usesAllOnesMask(SDValue MaskOp) {
4997 const auto IsVMSet = [](unsigned Opc) {
4998 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
4999 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
5000 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
5001 Opc == RISCV::PseudoVMSET_M_B8;
5002 };
5003
5004 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
5005 // undefined behaviour if it's the wrong bitwidth, so we could choose to
5006 // assume that it's all-ones? Same applies to its VL.
5007 return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode());
5008}
5009
5010static bool isImplicitDef(SDValue V) {
5011 if (!V.isMachineOpcode())
5012 return false;
5013 if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
5014 for (unsigned I = 1; I < V.getNumOperands(); I += 2)
5015 if (!isImplicitDef(V: V.getOperand(i: I)))
5016 return false;
5017 return true;
5018 }
5019 return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
5020}
5021
5022// Optimize masked RVV pseudo instructions with a known all-ones mask to their
5023// corresponding "unmasked" pseudo versions.
5024bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
5025 const RISCV::RISCVMaskedPseudoInfo *I =
5026 RISCV::getMaskedPseudoInfo(MaskedPseudo: N->getMachineOpcode());
5027 if (!I)
5028 return false;
5029
5030 unsigned MaskOpIdx = I->MaskOpIdx;
5031 if (!usesAllOnesMask(MaskOp: N->getOperand(Num: MaskOpIdx)))
5032 return false;
5033
5034 // There are two classes of pseudos in the table - compares and
5035 // everything else. See the comment on RISCVMaskedPseudo for details.
5036 const unsigned Opc = I->UnmaskedPseudo;
5037 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
5038 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MCID);
5039
5040 const MCInstrDesc &MaskedMCID = TII->get(Opcode: N->getMachineOpcode());
5041 const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MaskedMCID);
5042
5043 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
5044 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&
5045 "Unmasked pseudo has policy but masked pseudo doesn't?");
5046 assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru &&
5047 "Unexpected pseudo structure");
5048 assert(!(HasPassthru && !MaskedHasPassthru) &&
5049 "Unmasked pseudo has passthru but masked pseudo doesn't?");
5050
5051 SmallVector<SDValue, 8> Ops;
5052 // Skip the passthru operand at index 0 if the unmasked don't have one.
5053 bool ShouldSkip = !HasPassthru && MaskedHasPassthru;
5054 bool DropPolicy = !RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags) &&
5055 RISCVII::hasVecPolicyOp(TSFlags: MaskedMCID.TSFlags);
5056 bool HasChainOp =
5057 N->getOperand(Num: N->getNumOperands() - 1).getValueType() == MVT::Other;
5058 unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp;
5059 for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) {
5060 // Skip the mask
5061 SDValue Op = N->getOperand(Num: I);
5062 if (I == MaskOpIdx)
5063 continue;
5064 if (DropPolicy && I == LastOpNum)
5065 continue;
5066 Ops.push_back(Elt: Op);
5067 }
5068
5069 MachineSDNode *Result =
5070 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5071
5072 if (!N->memoperands_empty())
5073 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: N->memoperands());
5074
5075 Result->setFlags(N->getFlags());
5076 ReplaceUses(F: N, T: Result);
5077
5078 return true;
5079}
5080
5081/// If our passthru is an implicit_def, use noreg instead. This side
5082/// steps issues with MachineCSE not being able to CSE expressions with
5083/// IMPLICIT_DEF operands while preserving the semantic intent. See
5084/// pr64282 for context. Note that this transform is the last one
5085/// performed at ISEL DAG to DAG.
5086bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
5087 bool MadeChange = false;
5088 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5089
5090 while (Position != CurDAG->allnodes_begin()) {
5091 SDNode *N = &*--Position;
5092 if (N->use_empty() || !N->isMachineOpcode())
5093 continue;
5094
5095 const unsigned Opc = N->getMachineOpcode();
5096 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opc) ||
5097 !RISCVII::isFirstDefTiedToFirstUse(Desc: TII->get(Opcode: Opc)) ||
5098 !isImplicitDef(V: N->getOperand(Num: 0)))
5099 continue;
5100
5101 SmallVector<SDValue> Ops;
5102 Ops.push_back(Elt: CurDAG->getRegister(Reg: RISCV::NoRegister, VT: N->getValueType(ResNo: 0)));
5103 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
5104 SDValue Op = N->getOperand(Num: I);
5105 Ops.push_back(Elt: Op);
5106 }
5107
5108 MachineSDNode *Result =
5109 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5110 Result->setFlags(N->getFlags());
5111 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: cast<MachineSDNode>(Val: N)->memoperands());
5112 ReplaceUses(F: N, T: Result);
5113 MadeChange = true;
5114 }
5115 return MadeChange;
5116}
5117
5118
5119// This pass converts a legalized DAG into a RISCV-specific DAG, ready
5120// for instruction scheduling.
5121FunctionPass *llvm::createRISCVISelDagLegacyPass(RISCVTargetMachine &TM,
5122 CodeGenOptLevel OptLevel) {
5123 return new RISCVDAGToDAGISelLegacy(TM, OptLevel);
5124}
5125
5126RISCVISelDAGToDAGPass::RISCVISelDAGToDAGPass(RISCVTargetMachine &TM,
5127 CodeGenOptLevel OptLevel)
5128 : SelectionDAGISelPass(std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5129
5130char RISCVDAGToDAGISelLegacy::ID = 0;
5131
5132RISCVDAGToDAGISelLegacy::RISCVDAGToDAGISelLegacy(RISCVTargetMachine &TM,
5133 CodeGenOptLevel OptLevel)
5134 : SelectionDAGISelLegacy(
5135 ID, std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5136
5137INITIALIZE_PASS(RISCVDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
5138