1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "MCTargetDesc/RISCVMatInt.h"
17#include "RISCVISelLowering.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSelectionDAGInfo.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/Support/Alignment.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
32extern cl::opt<uint32_t> PreferredLandingPadLabel;
33
34static cl::opt<bool> UsePseudoMovImm(
35 "riscv-use-rematerializable-movimm", cl::Hidden,
36 cl::desc("Use a rematerializable pseudoinstruction for 2 instruction "
37 "constant materialization"),
38 cl::init(Val: false));
39
40#define GET_DAGISEL_BODY RISCVDAGToDAGISel
41#include "RISCVGenDAGISel.inc"
42
43void RISCVDAGToDAGISel::PreprocessISelDAG() {
44 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
45
46 bool MadeChange = false;
47 while (Position != CurDAG->allnodes_begin()) {
48 SDNode *N = &*--Position;
49 if (N->use_empty())
50 continue;
51
52 SDValue Result;
53 switch (N->getOpcode()) {
54 case ISD::SPLAT_VECTOR: {
55 if (Subtarget->hasStdExtP())
56 break;
57 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
58 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
59 MVT VT = N->getSimpleValueType(ResNo: 0);
60 unsigned Opc =
61 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
62 SDLoc DL(N);
63 SDValue VL = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
64 SDValue Src = N->getOperand(Num: 0);
65 if (VT.isInteger())
66 Src = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget->getXLenVT(),
67 Operand: N->getOperand(Num: 0));
68 Result = CurDAG->getNode(Opcode: Opc, DL, VT, N1: CurDAG->getUNDEF(VT), N2: Src, N3: VL);
69 break;
70 }
71 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
72 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
73 // load. Done after lowering and combining so that we have a chance to
74 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
75 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
76 MVT VT = N->getSimpleValueType(ResNo: 0);
77 SDValue Passthru = N->getOperand(Num: 0);
78 SDValue Lo = N->getOperand(Num: 1);
79 SDValue Hi = N->getOperand(Num: 2);
80 SDValue VL = N->getOperand(Num: 3);
81 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
82 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
83 "Unexpected VTs!");
84 MachineFunction &MF = CurDAG->getMachineFunction();
85 SDLoc DL(N);
86
87 // Create temporary stack for each expanding node.
88 SDValue StackSlot =
89 CurDAG->CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 8), Alignment: Align(8));
90 int FI = cast<FrameIndexSDNode>(Val: StackSlot.getNode())->getIndex();
91 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
92
93 SDValue Chain = CurDAG->getEntryNode();
94 Lo = CurDAG->getStore(Chain, dl: DL, Val: Lo, Ptr: StackSlot, PtrInfo: MPI, Alignment: Align(8));
95
96 SDValue OffsetSlot =
97 CurDAG->getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL);
98 Hi = CurDAG->getStore(Chain, dl: DL, Val: Hi, Ptr: OffsetSlot, PtrInfo: MPI.getWithOffset(O: 4),
99 Alignment: Align(8));
100
101 Chain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
102
103 SDVTList VTs = CurDAG->getVTList(VTs: {VT, MVT::Other});
104 SDValue IntID =
105 CurDAG->getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: MVT::i64);
106 SDValue Ops[] = {Chain,
107 IntID,
108 Passthru,
109 StackSlot,
110 CurDAG->getRegister(Reg: RISCV::X0, VT: MVT::i64),
111 VL};
112
113 Result = CurDAG->getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
114 MemVT: MVT::i64, PtrInfo: MPI, Alignment: Align(8),
115 Flags: MachineMemOperand::MOLoad);
116 break;
117 }
118 case ISD::FP_EXTEND: {
119 // We only have vector patterns for riscv_fpextend_vl in isel.
120 SDLoc DL(N);
121 MVT VT = N->getSimpleValueType(ResNo: 0);
122 if (!VT.isVector())
123 break;
124 SDValue VLMAX = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
125 SDValue TrueMask = CurDAG->getNode(
126 Opcode: RISCVISD::VMSET_VL, DL, VT: VT.changeVectorElementType(EltVT: MVT::i1), Operand: VLMAX);
127 Result = CurDAG->getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT, N1: N->getOperand(Num: 0),
128 N2: TrueMask, N3: VLMAX);
129 break;
130 }
131 case ISD::ADD: {
132 // Turn (add X, C) into (sub X, -C) when a constant node holding -C
133 // already exists in the DAG, so both share one materialization. Do this
134 // before selection, while both are still ConstantSDNodes: by selection
135 // time -C may already have been selected into instructions.
136 //
137 // ADD is commutative, but getNode canonicalizes constants to the RHS, so
138 // the constant is always operand 1.
139 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
140 if (!N1C)
141 break;
142 MVT VT = N->getSimpleValueType(ResNo: 0);
143 if (VT != Subtarget->getXLenVT())
144 break;
145 int64_t Imm = N1C->getSExtValue();
146 // Only worthwhile for wide constants: values that fit in 32 bits take at
147 // most two instructions to materialize, matching the threshold used by
148 // selectNegImm. Skip INT64_MIN too, whose negation is itself.
149 if (isInt<32>(x: Imm) || Imm == INT64_MIN)
150 break;
151 // A constant is anchored if it has a user other than an ADD, i.e. it is
152 // materialized regardless of this fold. N1C is the (unique) node for Imm,
153 // so the positive side needs no search.
154 auto IsAnchored = [](const SDNode *C) {
155 return any_of(Range: C->users(), P: [](const SDNode *U) {
156 return U->getOpcode() != ISD::ADD;
157 });
158 };
159 // If Imm is materialized anyway, keep the ADD so it reuses Imm; an ADD is
160 // also more compressible than a SUB. This also lets us skip the search
161 // for -Imm below.
162 if (IsAnchored(N1C))
163 break;
164 // Find the (unique) constant node for -Imm, if any.
165 const SDNode *NegC = nullptr;
166 for (const SDNode &Node : CurDAG->allnodes()) {
167 auto *C = dyn_cast<ConstantSDNode>(Val: &Node);
168 if (C && C->getSimpleValueType(ResNo: 0) == VT && C->getSExtValue() == -Imm) {
169 NegC = &Node;
170 break;
171 }
172 }
173 // Reuse is only free if -Imm is already in the DAG.
174 if (!NegC)
175 break;
176 // dyn_cast<ConstantSDNode> also matches TargetConstant, which is encoded
177 // into the instruction rather than materialized, so reusing it would not
178 // remove a materialization. No TargetConstant is this wide (the largest
179 // are intrinsic IDs, which fit in 32 bits), so assert it is a Constant.
180 assert(NegC->getOpcode() == ISD::Constant &&
181 "Unexpected wide TargetConstant");
182 // Pick which of Imm/-Imm should be the surviving constant, so exactly
183 // one of the pair is materialized and any ADDs of the other reuse it:
184 // - if -Imm is materialized anyway, reuse it (rewrite to SUB);
185 // - else keep the cheaper constant, breaking ties towards the positive
186 // value so both ADDs of a C/-C pair agree on the survivor.
187 bool Rewrite;
188 if (IsAnchored(NegC)) {
189 Rewrite = true;
190 } else {
191 int PosCost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
192 /*CompressionCost=*/true);
193 int NegCost =
194 RISCVMatInt::getIntMatCost(Val: APInt(64, -Imm), Size: 64, STI: *Subtarget,
195 /*CompressionCost=*/true);
196 Rewrite = NegCost != PosCost ? NegCost < PosCost : Imm < 0;
197 }
198 if (!Rewrite)
199 break;
200 SDLoc DL(N);
201 // getConstant uniques onto the existing -C node, so it is shared.
202 Result = CurDAG->getNode(Opcode: ISD::SUB, DL, VT, N1: N->getOperand(Num: 0),
203 N2: CurDAG->getConstant(Val: -Imm, DL, VT));
204 break;
205 }
206 }
207
208 if (Result) {
209 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
210 LLVM_DEBUG(N->dump(CurDAG));
211 LLVM_DEBUG(dbgs() << "\nNew: ");
212 LLVM_DEBUG(Result->dump(CurDAG));
213 LLVM_DEBUG(dbgs() << "\n");
214
215 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
216 MadeChange = true;
217 }
218 }
219
220 if (MadeChange)
221 CurDAG->RemoveDeadNodes();
222}
223
224void RISCVDAGToDAGISel::PostprocessISelDAG() {
225 HandleSDNode Dummy(CurDAG->getRoot());
226 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
227
228 bool MadeChange = false;
229 while (Position != CurDAG->allnodes_begin()) {
230 SDNode *N = &*--Position;
231 // Skip dead nodes and any non-machine opcodes.
232 if (N->use_empty() || !N->isMachineOpcode())
233 continue;
234
235 MadeChange |= doPeepholeSExtW(Node: N);
236
237 // FIXME: This is here only because the VMerge transform doesn't
238 // know how to handle masked true inputs. Once that has been moved
239 // to post-ISEL, this can be deleted as well.
240 MadeChange |= doPeepholeMaskedRVV(Node: cast<MachineSDNode>(Val: N));
241 }
242
243 CurDAG->setRoot(Dummy.getValue());
244
245 // After we're done with everything else, convert IMPLICIT_DEF
246 // passthru operands to NoRegister. This is required to workaround
247 // an optimization deficiency in MachineCSE. This really should
248 // be merged back into each of the patterns (i.e. there's no good
249 // reason not to go directly to NoReg), but is being done this way
250 // to allow easy backporting.
251 MadeChange |= doPeepholeNoRegPassThru();
252
253 if (MadeChange)
254 CurDAG->RemoveDeadNodes();
255}
256
257static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
258 RISCVMatInt::InstSeq &Seq) {
259 SDValue SrcReg = CurDAG->getRegister(Reg: RISCV::X0, VT);
260 for (const RISCVMatInt::Inst &Inst : Seq) {
261 SDValue SDImm = CurDAG->getSignedTargetConstant(Val: Inst.getImm(), DL, VT);
262 SDNode *Result = nullptr;
263 switch (Inst.getOpndKind()) {
264 case RISCVMatInt::Imm:
265 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SDImm);
266 break;
267 case RISCVMatInt::RegX0:
268 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg,
269 Op2: CurDAG->getRegister(Reg: RISCV::X0, VT));
270 break;
271 case RISCVMatInt::RegReg:
272 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SrcReg);
273 break;
274 case RISCVMatInt::RegImm:
275 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SDImm);
276 break;
277 }
278
279 // Only the first instruction has X0 as its source.
280 SrcReg = SDValue(Result, 0);
281 }
282
283 return SrcReg;
284}
285
286static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
287 int64_t Imm, const RISCVSubtarget &Subtarget) {
288 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
289
290 // Use a rematerializable pseudo instruction for short sequences if enabled.
291 if (Seq.size() == 2 && UsePseudoMovImm)
292 return SDValue(
293 CurDAG->getMachineNode(Opcode: RISCV::PseudoMovImm, dl: DL, VT,
294 Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT)),
295 0);
296
297 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
298 // worst an LUI+ADDIW. This will require an extra register, but avoids a
299 // constant pool.
300 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
301 // low and high 32 bits are the same and bit 31 and 63 are set.
302 if (Seq.size() > 3) {
303 unsigned ShiftAmt, AddOpc;
304 RISCVMatInt::InstSeq SeqLo =
305 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
306 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
307 SDValue Lo = selectImmSeq(CurDAG, DL, VT, Seq&: SeqLo);
308
309 SDValue SLLI = SDValue(
310 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: Lo,
311 Op2: CurDAG->getTargetConstant(Val: ShiftAmt, DL, VT)),
312 0);
313 return SDValue(CurDAG->getMachineNode(Opcode: AddOpc, dl: DL, VT, Op1: Lo, Op2: SLLI), 0);
314 }
315 }
316
317 // Otherwise, use the original sequence.
318 return selectImmSeq(CurDAG, DL, VT, Seq);
319}
320
321void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
322 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
323 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
324 bool IsLoad, MVT *IndexVT) {
325 SDValue Chain = Node->getOperand(Num: 0);
326
327 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Base pointer.
328
329 if (IsStridedOrIndexed) {
330 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Index.
331 if (IndexVT)
332 *IndexVT = Operands.back()->getSimpleValueType(ResNo: 0);
333 }
334
335 if (IsMasked) {
336 SDValue Mask = Node->getOperand(Num: CurOp++);
337 Operands.push_back(Elt: Mask);
338 }
339 SDValue VL;
340 selectVLOp(N: Node->getOperand(Num: CurOp++), VL);
341 Operands.push_back(Elt: VL);
342
343 MVT XLenVT = Subtarget->getXLenVT();
344 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
345 Operands.push_back(Elt: SEWOp);
346
347 // At the IR layer, all the masked load intrinsics have policy operands,
348 // none of the others do. All have passthru operands. For our pseudos,
349 // all loads have policy operands.
350 if (IsLoad) {
351 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
352 if (IsMasked)
353 Policy = Node->getConstantOperandVal(Num: CurOp++);
354 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
355 Operands.push_back(Elt: PolicyOp);
356 }
357
358 Operands.push_back(Elt: Chain); // Chain.
359}
360
361void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked,
362 bool IsStrided) {
363 SDLoc DL(Node);
364 MVT VT = Node->getSimpleValueType(ResNo: 0);
365 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
366 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
367
368 unsigned CurOp = 2;
369 SmallVector<SDValue, 8> Operands;
370
371 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
372
373 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
374 Operands, /*IsLoad=*/true);
375
376 const RISCV::VLSEGPseudo *P =
377 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
378 LMUL: static_cast<unsigned>(LMUL));
379 MachineSDNode *Load =
380 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
381
382 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
383
384 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
385 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
386 CurDAG->RemoveDeadNode(N: Node);
387}
388
389void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, unsigned NF,
390 bool IsMasked) {
391 SDLoc DL(Node);
392 MVT VT = Node->getSimpleValueType(ResNo: 0);
393 MVT XLenVT = Subtarget->getXLenVT();
394 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
395 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
396
397 unsigned CurOp = 2;
398 SmallVector<SDValue, 7> Operands;
399
400 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
401
402 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
403 /*IsStridedOrIndexed*/ false, Operands,
404 /*IsLoad=*/true);
405
406 const RISCV::VLSEGPseudo *P =
407 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
408 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
409 MachineSDNode *Load = CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped,
410 VT2: XLenVT, VT3: MVT::Other, Ops: Operands);
411
412 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
413
414 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0)); // Result
415 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1)); // VL
416 ReplaceUses(F: SDValue(Node, 2), T: SDValue(Load, 2)); // Chain
417 CurDAG->RemoveDeadNode(N: Node);
418}
419
420void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked,
421 bool IsOrdered) {
422 SDLoc DL(Node);
423 MVT VT = Node->getSimpleValueType(ResNo: 0);
424 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
425 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
426
427 unsigned CurOp = 2;
428 SmallVector<SDValue, 8> Operands;
429
430 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
431
432 MVT IndexVT;
433 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
434 /*IsStridedOrIndexed*/ true, Operands,
435 /*IsLoad=*/true, IndexVT: &IndexVT);
436
437#ifndef NDEBUG
438 // Number of element = RVVBitsPerBlock * LMUL / SEW
439 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
440 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
441 if (DecodedLMUL.second)
442 ContainedTyNumElts /= DecodedLMUL.first;
443 else
444 ContainedTyNumElts *= DecodedLMUL.first;
445 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
446 "Element count mismatch");
447#endif
448
449 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
450 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
451 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
452 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
453 "values when XLEN=32");
454 }
455 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
456 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
457 IndexLMUL: static_cast<unsigned>(IndexLMUL));
458 MachineSDNode *Load =
459 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
460
461 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
462
463 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
464 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
465 CurDAG->RemoveDeadNode(N: Node);
466}
467
468void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked,
469 bool IsStrided) {
470 SDLoc DL(Node);
471 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
472 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
473 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
474
475 unsigned CurOp = 2;
476 SmallVector<SDValue, 8> Operands;
477
478 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
479
480 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
481 Operands);
482
483 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
484 NF, Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
485 MachineSDNode *Store =
486 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
487
488 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
489
490 ReplaceNode(F: Node, T: Store);
491}
492
493void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked,
494 bool IsOrdered) {
495 SDLoc DL(Node);
496 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
497 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
498 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
499
500 unsigned CurOp = 2;
501 SmallVector<SDValue, 8> Operands;
502
503 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
504
505 MVT IndexVT;
506 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
507 /*IsStridedOrIndexed*/ true, Operands,
508 /*IsLoad=*/false, IndexVT: &IndexVT);
509
510#ifndef NDEBUG
511 // Number of element = RVVBitsPerBlock * LMUL / SEW
512 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
513 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
514 if (DecodedLMUL.second)
515 ContainedTyNumElts /= DecodedLMUL.first;
516 else
517 ContainedTyNumElts *= DecodedLMUL.first;
518 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
519 "Element count mismatch");
520#endif
521
522 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
523 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
524 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
525 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
526 "values when XLEN=32");
527 }
528 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
529 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
530 IndexLMUL: static_cast<unsigned>(IndexLMUL));
531 MachineSDNode *Store =
532 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
533
534 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
535
536 ReplaceNode(F: Node, T: Store);
537}
538
539void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) {
540 if (!Subtarget->hasVInstructions())
541 return;
542
543 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
544
545 SDLoc DL(Node);
546 MVT XLenVT = Subtarget->getXLenVT();
547
548 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
549
550 assert((IntNo == Intrinsic::riscv_vsetvli ||
551 IntNo == Intrinsic::riscv_vsetvlimax) &&
552 "Unexpected vsetvli intrinsic");
553
554 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
555 unsigned Offset = (VLMax ? 1 : 2);
556
557 assert(Node->getNumOperands() == Offset + 2 &&
558 "Unexpected number of operands");
559
560 unsigned SEW =
561 RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: Offset) & 0x7);
562 RISCVVType::VLMUL VLMul = static_cast<RISCVVType::VLMUL>(
563 Node->getConstantOperandVal(Num: Offset + 1) & 0x7);
564
565 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
566 /*MaskAgnostic*/ true);
567 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
568
569 SDValue VLOperand;
570 unsigned Opcode = RISCV::PseudoVSETVLI;
571 if (auto *C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
572 if (auto VLEN = Subtarget->getRealVLen())
573 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
574 VLMax = true;
575 }
576 if (VLMax || isAllOnesConstant(V: Node->getOperand(Num: 1))) {
577 VLOperand = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
578 Opcode = RISCV::PseudoVSETVLIX0;
579 } else {
580 VLOperand = Node->getOperand(Num: 1);
581
582 if (auto *C = dyn_cast<ConstantSDNode>(Val&: VLOperand)) {
583 uint64_t AVL = C->getZExtValue();
584 if (isUInt<5>(x: AVL)) {
585 SDValue VLImm = CurDAG->getTargetConstant(Val: AVL, DL, VT: XLenVT);
586 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: RISCV::PseudoVSETIVLI, dl: DL,
587 VT: XLenVT, Op1: VLImm, Op2: VTypeIOp));
588 return;
589 }
590 }
591 }
592
593 ReplaceNode(F: Node,
594 T: CurDAG->getMachineNode(Opcode, dl: DL, VT: XLenVT, Op1: VLOperand, Op2: VTypeIOp));
595}
596
597void RISCVDAGToDAGISel::selectXSfmmVSET(SDNode *Node) {
598 if (!Subtarget->hasVendorXSfmmbase())
599 return;
600
601 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
602
603 SDLoc DL(Node);
604 MVT XLenVT = Subtarget->getXLenVT();
605
606 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
607
608 assert((IntNo == Intrinsic::riscv_sf_vsettnt ||
609 IntNo == Intrinsic::riscv_sf_vsettm ||
610 IntNo == Intrinsic::riscv_sf_vsettk) &&
611 "Unexpected XSfmm vset intrinsic");
612
613 unsigned SEW = RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: 2));
614 unsigned Widen = RISCVVType::decodeTWiden(TWiden: Node->getConstantOperandVal(Num: 3));
615 unsigned PseudoOpCode =
616 IntNo == Intrinsic::riscv_sf_vsettnt ? RISCV::PseudoSF_VSETTNT
617 : IntNo == Intrinsic::riscv_sf_vsettm ? RISCV::PseudoSF_VSETTM
618 : RISCV::PseudoSF_VSETTK;
619
620 if (IntNo == Intrinsic::riscv_sf_vsettnt) {
621 unsigned VTypeI = RISCVVType::encodeXSfmmVType(SEW, Widen, AltFmt: 0);
622 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
623
624 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
625 Op1: Node->getOperand(Num: 1), Op2: VTypeIOp));
626 } else {
627 SDValue Log2SEW = CurDAG->getTargetConstant(Val: Log2_32(Value: SEW), DL, VT: XLenVT);
628 SDValue TWiden = CurDAG->getTargetConstant(Val: Widen, DL, VT: XLenVT);
629 ReplaceNode(F: Node,
630 T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
631 Op1: Node->getOperand(Num: 1), Op2: Log2SEW, Op3: TWiden));
632 }
633}
634
635bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
636 MVT VT = Node->getSimpleValueType(ResNo: 0);
637 unsigned Opcode = Node->getOpcode();
638 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
639 "Unexpected opcode");
640 SDLoc DL(Node);
641
642 // For operations of the form (x << C1) op C2, check if we can use
643 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
644 SDValue N0 = Node->getOperand(Num: 0);
645 SDValue N1 = Node->getOperand(Num: 1);
646
647 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
648 if (!Cst)
649 return false;
650
651 int64_t Val = Cst->getSExtValue();
652
653 // Check if immediate can already use ANDI/ORI/XORI.
654 if (isInt<12>(x: Val))
655 return false;
656
657 SDValue Shift = N0;
658
659 // If Val is simm32 and we have a sext_inreg from i32, then the binop
660 // produces at least 33 sign bits. We can peek through the sext_inreg and use
661 // a SLLIW at the end.
662 bool SignExt = false;
663 if (isInt<32>(x: Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
664 N0.hasOneUse() && cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT() == MVT::i32) {
665 SignExt = true;
666 Shift = N0.getOperand(i: 0);
667 }
668
669 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
670 return false;
671
672 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1));
673 if (!ShlCst)
674 return false;
675
676 uint64_t ShAmt = ShlCst->getZExtValue();
677
678 // Make sure that we don't change the operation by removing bits.
679 // This only matters for OR and XOR, AND is unaffected.
680 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(N: ShAmt);
681 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
682 return false;
683
684 int64_t ShiftedVal = Val >> ShAmt;
685 if (!isInt<12>(x: ShiftedVal))
686 return false;
687
688 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
689 if (SignExt && ShAmt >= 32)
690 return false;
691
692 // Ok, we can reorder to get a smaller immediate.
693 unsigned BinOpc;
694 switch (Opcode) {
695 default: llvm_unreachable("Unexpected opcode");
696 case ISD::AND: BinOpc = RISCV::ANDI; break;
697 case ISD::OR: BinOpc = RISCV::ORI; break;
698 case ISD::XOR: BinOpc = RISCV::XORI; break;
699 }
700
701 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
702
703 SDNode *BinOp = CurDAG->getMachineNode(
704 Opcode: BinOpc, dl: DL, VT, Op1: Shift.getOperand(i: 0),
705 Op2: CurDAG->getSignedTargetConstant(Val: ShiftedVal, DL, VT));
706 SDNode *SLLI =
707 CurDAG->getMachineNode(Opcode: ShOpc, dl: DL, VT, Op1: SDValue(BinOp, 0),
708 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
709 ReplaceNode(F: Node, T: SLLI);
710 return true;
711}
712
713bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
714 unsigned Opc;
715
716 if (Subtarget->hasVendorXTHeadBb())
717 Opc = RISCV::TH_EXT;
718 else if (Subtarget->hasVendorXAndesPerf())
719 Opc = RISCV::NDS_BFOS;
720 else if (Subtarget->hasVendorXqcibm())
721 Opc = RISCV::QC_EXT;
722 else
723 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
724 return false;
725
726 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
727 if (!N1C)
728 return false;
729
730 SDValue N0 = Node->getOperand(Num: 0);
731 if (!N0.hasOneUse())
732 return false;
733
734 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
735 const SDLoc &DL, MVT VT) {
736 if (Opc == RISCV::QC_EXT) {
737 // QC.EXT X, width, shamt
738 // shamt is the same as Lsb
739 // width is the number of bits to extract from the Lsb
740 Msb = Msb - Lsb + 1;
741 }
742 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
743 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
744 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
745 };
746
747 SDLoc DL(Node);
748 MVT VT = Node->getSimpleValueType(ResNo: 0);
749 const unsigned RightShAmt = N1C->getZExtValue();
750
751 // Transform (sra (shl X, C1) C2) with C1 < C2
752 // -> (SignedBitfieldExtract X, msb, lsb)
753 if (N0.getOpcode() == ISD::SHL) {
754 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
755 if (!N01C)
756 return false;
757
758 const unsigned LeftShAmt = N01C->getZExtValue();
759 // Make sure that this is a bitfield extraction (i.e., the shift-right
760 // amount can not be less than the left-shift).
761 if (LeftShAmt > RightShAmt)
762 return false;
763
764 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
765 const unsigned Msb = MsbPlusOne - 1;
766 const unsigned Lsb = RightShAmt - LeftShAmt;
767
768 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
769 ReplaceNode(F: Node, T: Sbe);
770 return true;
771 }
772
773 // Transform (sra (sext_inreg X, _), C) ->
774 // (SignedBitfieldExtract X, msb, lsb)
775 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
776 unsigned ExtSize =
777 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
778
779 // ExtSize of 32 should use sraiw via tablegen pattern.
780 if (ExtSize == 32)
781 return false;
782
783 const unsigned Msb = ExtSize - 1;
784 // If the shift-right amount is greater than Msb, it means that extracts
785 // the X[Msb] bit and sign-extend it.
786 const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
787
788 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
789 ReplaceNode(F: Node, T: Sbe);
790 return true;
791 }
792
793 return false;
794}
795
796bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
797 // Only supported with XAndesPerf at the moment.
798 if (!Subtarget->hasVendorXAndesPerf())
799 return false;
800
801 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
802 if (!N1C)
803 return false;
804
805 SDValue N0 = Node->getOperand(Num: 0);
806 if (!N0.hasOneUse())
807 return false;
808
809 auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
810 const SDLoc &DL, MVT VT) {
811 unsigned Opc = RISCV::NDS_BFOS;
812 // If the Lsb is equal to the Msb, then the Lsb should be 0.
813 if (Lsb == Msb)
814 Lsb = 0;
815 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
816 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
817 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
818 };
819
820 SDLoc DL(Node);
821 MVT VT = Node->getSimpleValueType(ResNo: 0);
822 const unsigned RightShAmt = N1C->getZExtValue();
823
824 // Transform (sra (shl X, C1) C2) with C1 > C2
825 // -> (NDS.BFOS X, lsb, msb)
826 if (N0.getOpcode() == ISD::SHL) {
827 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
828 if (!N01C)
829 return false;
830
831 const unsigned LeftShAmt = N01C->getZExtValue();
832 // Make sure that this is a bitfield insertion (i.e., the shift-right
833 // amount should be less than the left-shift).
834 if (LeftShAmt <= RightShAmt)
835 return false;
836
837 const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
838 const unsigned Msb = MsbPlusOne - 1;
839 const unsigned Lsb = LeftShAmt - RightShAmt;
840
841 SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
842 ReplaceNode(F: Node, T: Sbi);
843 return true;
844 }
845
846 return false;
847}
848
849bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
850 const SDLoc &DL, MVT VT,
851 SDValue X, unsigned Msb,
852 unsigned Lsb) {
853 unsigned Opc;
854
855 if (Subtarget->hasVendorXTHeadBb()) {
856 Opc = RISCV::TH_EXTU;
857 } else if (Subtarget->hasVendorXAndesPerf()) {
858 Opc = RISCV::NDS_BFOZ;
859 } else if (Subtarget->hasVendorXqcibm()) {
860 Opc = RISCV::QC_EXTU;
861 // QC.EXTU X, width, shamt
862 // shamt is the same as Lsb
863 // width is the number of bits to extract from the Lsb
864 Msb = Msb - Lsb + 1;
865 } else {
866 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
867 return false;
868 }
869
870 SDNode *Ube = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
871 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
872 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
873 ReplaceNode(F: Node, T: Ube);
874 return true;
875}
876
877bool RISCVDAGToDAGISel::tryUnsignedBitfieldInsertInZero(SDNode *Node,
878 const SDLoc &DL, MVT VT,
879 SDValue X, unsigned Msb,
880 unsigned Lsb) {
881 // Only supported with XAndesPerf at the moment.
882 if (!Subtarget->hasVendorXAndesPerf())
883 return false;
884
885 unsigned Opc = RISCV::NDS_BFOZ;
886
887 // If the Lsb is equal to the Msb, then the Lsb should be 0.
888 if (Lsb == Msb)
889 Lsb = 0;
890 SDNode *Ubi = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
891 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
892 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
893 ReplaceNode(F: Node, T: Ubi);
894 return true;
895}
896
897bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
898 // Target does not support indexed loads.
899 if (!Subtarget->hasVendorXTHeadMemIdx())
900 return false;
901
902 LoadSDNode *Ld = cast<LoadSDNode>(Val: Node);
903 ISD::MemIndexedMode AM = Ld->getAddressingMode();
904 if (AM == ISD::UNINDEXED)
905 return false;
906
907 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Ld->getOffset());
908 if (!C)
909 return false;
910
911 EVT LoadVT = Ld->getMemoryVT();
912 assert((AM == ISD::PRE_INC || AM == ISD::POST_INC) &&
913 "Unexpected addressing mode");
914 bool IsPre = AM == ISD::PRE_INC;
915 bool IsPost = AM == ISD::POST_INC;
916 int64_t Offset = C->getSExtValue();
917
918 // The constants that can be encoded in the THeadMemIdx instructions
919 // are of the form (sign_extend(imm5) << imm2).
920 unsigned Shift;
921 for (Shift = 0; Shift < 4; Shift++)
922 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
923 break;
924
925 // Constant cannot be encoded.
926 if (Shift == 4)
927 return false;
928
929 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
930 unsigned Opcode;
931 if (LoadVT == MVT::i8 && IsPre)
932 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
933 else if (LoadVT == MVT::i8 && IsPost)
934 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
935 else if (LoadVT == MVT::i16 && IsPre)
936 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
937 else if (LoadVT == MVT::i16 && IsPost)
938 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
939 else if (LoadVT == MVT::i32 && IsPre)
940 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
941 else if (LoadVT == MVT::i32 && IsPost)
942 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
943 else if (LoadVT == MVT::i64 && IsPre)
944 Opcode = RISCV::TH_LDIB;
945 else if (LoadVT == MVT::i64 && IsPost)
946 Opcode = RISCV::TH_LDIA;
947 else
948 return false;
949
950 EVT Ty = Ld->getOffset().getValueType();
951 SDValue Ops[] = {
952 Ld->getBasePtr(),
953 CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(Node), VT: Ty),
954 CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(Node), VT: Ty), Ld->getChain()};
955 SDNode *New = CurDAG->getMachineNode(Opcode, dl: SDLoc(Node), VT1: Ld->getValueType(ResNo: 0),
956 VT2: Ld->getValueType(ResNo: 1), VT3: MVT::Other, Ops);
957
958 MachineMemOperand *MemOp = cast<MemSDNode>(Val: Node)->getMemOperand();
959 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: New), NewMemRefs: {MemOp});
960
961 ReplaceNode(F: Node, T: New);
962
963 return true;
964}
965
966static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT,
967 SDValue Lo, SDValue Hi) {
968 SDValue Ops[] = {
969 CurDAG->getTargetConstant(Val: RISCV::GPRPairRegClassID, DL, VT: MVT::i32), Lo,
970 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_even, DL, VT: MVT::i32), Hi,
971 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_odd, DL, VT: MVT::i32)};
972
973 return SDValue(
974 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT, Ops), 0);
975}
976
977// Helper to extract Lo and Hi values from a GPR pair.
978static std::pair<SDValue, SDValue>
979extractGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, SDValue Pair) {
980 SDValue Lo =
981 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL, VT: MVT::i32, Operand: Pair);
982 SDValue Hi =
983 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_odd, DL, VT: MVT::i32, Operand: Pair);
984 return {Lo, Hi};
985}
986
987// Try to match WMACC pattern: ADDD where one operand pair comes from a
988// widening multiply (both results of UMUL_LOHI, SMUL_LOHI, or WMULSU).
989bool RISCVDAGToDAGISel::tryWideningMulAcc(SDNode *Node, const SDLoc &DL) {
990 assert(Node->getOpcode() == RISCVISD::ADDD && "Expected ADDD");
991
992 SDValue Op0Lo = Node->getOperand(Num: 0);
993 SDValue Op0Hi = Node->getOperand(Num: 1);
994 SDValue Op1Lo = Node->getOperand(Num: 2);
995 SDValue Op1Hi = Node->getOperand(Num: 3);
996
997 auto IsSupportedMulWithOneUse = [](SDValue Lo, SDValue Hi) {
998 unsigned Opc = Lo.getOpcode();
999 if (Opc != ISD::UMUL_LOHI && Opc != ISD::SMUL_LOHI &&
1000 Opc != RISCVISD::WMULSU)
1001 return false;
1002 return Lo.getNode() == Hi.getNode() && Lo.getResNo() == 0 &&
1003 Hi.getResNo() == 1 && Lo.hasOneUse() && Hi.hasOneUse();
1004 };
1005
1006 SDNode *MulNode = nullptr;
1007 SDValue AddLo, AddHi;
1008
1009 // Check if first operand pair is a supported multiply with single use.
1010 if (IsSupportedMulWithOneUse(Op0Lo, Op0Hi)) {
1011 MulNode = Op0Lo.getNode();
1012 AddLo = Op1Lo;
1013 AddHi = Op1Hi;
1014 }
1015 // ADDD is commutative. Check if second operand pair is a supported multiply
1016 // with single use.
1017 else if (IsSupportedMulWithOneUse(Op1Lo, Op1Hi)) {
1018 MulNode = Op1Lo.getNode();
1019 AddLo = Op0Lo;
1020 AddHi = Op0Hi;
1021 } else {
1022 return false;
1023 }
1024
1025 unsigned Opc;
1026 switch (MulNode->getOpcode()) {
1027 default:
1028 llvm_unreachable("Unexpected multiply opcode");
1029 case ISD::UMUL_LOHI:
1030 Opc = RISCV::WMACCU;
1031 break;
1032 case ISD::SMUL_LOHI:
1033 Opc = RISCV::WMACC;
1034 break;
1035 case RISCVISD::WMULSU:
1036 Opc = RISCV::WMACCSU;
1037 break;
1038 }
1039
1040 SDValue Acc = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: AddLo, Hi: AddHi);
1041
1042 // WMACC instruction format: rd, rs1, rs2 (rd is accumulator).
1043 SDValue M0 = MulNode->getOperand(Num: 0);
1044 SDValue M1 = MulNode->getOperand(Num: 1);
1045 MachineSDNode *New =
1046 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Acc, Op2: M0, Op3: M1);
1047
1048 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
1049 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1050 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1051 CurDAG->RemoveDeadNode(N: Node);
1052 return true;
1053}
1054
1055static Register getTileReg(uint64_t TileNum) {
1056 assert(TileNum <= 15 && "Invalid tile number");
1057 return RISCV::T0 + TileNum;
1058}
1059
1060void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
1061 if (!Subtarget->hasVInstructions())
1062 return;
1063
1064 assert(Node->getOpcode() == ISD::INTRINSIC_VOID && "Unexpected opcode");
1065
1066 SDLoc DL(Node);
1067 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
1068
1069 assert((IntNo == Intrinsic::riscv_sf_vc_x_se ||
1070 IntNo == Intrinsic::riscv_sf_vc_i_se) &&
1071 "Unexpected vsetvli intrinsic");
1072
1073 // imm, imm, imm, simm5/scalar, sew, log2lmul, vl
1074 unsigned Log2SEW = Log2_32(Value: Node->getConstantOperandVal(Num: 6));
1075 SDValue SEWOp =
1076 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: Subtarget->getXLenVT());
1077 SmallVector<SDValue, 8> Operands = {Node->getOperand(Num: 2), Node->getOperand(Num: 3),
1078 Node->getOperand(Num: 4), Node->getOperand(Num: 5),
1079 Node->getOperand(Num: 8), SEWOp,
1080 Node->getOperand(Num: 0)};
1081
1082 unsigned Opcode;
1083 auto *LMulSDNode = cast<ConstantSDNode>(Val: Node->getOperand(Num: 7));
1084 switch (LMulSDNode->getSExtValue()) {
1085 case 5:
1086 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
1087 : RISCV::PseudoSF_VC_I_SE_MF8;
1088 break;
1089 case 6:
1090 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
1091 : RISCV::PseudoSF_VC_I_SE_MF4;
1092 break;
1093 case 7:
1094 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
1095 : RISCV::PseudoSF_VC_I_SE_MF2;
1096 break;
1097 case 0:
1098 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
1099 : RISCV::PseudoSF_VC_I_SE_M1;
1100 break;
1101 case 1:
1102 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
1103 : RISCV::PseudoSF_VC_I_SE_M2;
1104 break;
1105 case 2:
1106 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
1107 : RISCV::PseudoSF_VC_I_SE_M4;
1108 break;
1109 case 3:
1110 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
1111 : RISCV::PseudoSF_VC_I_SE_M8;
1112 break;
1113 }
1114
1115 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
1116 Opcode, dl: DL, VT: Node->getSimpleValueType(ResNo: 0), Ops: Operands));
1117}
1118
1119static unsigned getSegInstNF(unsigned Intrinsic) {
1120#define INST_NF_CASE(NAME, NF) \
1121 case Intrinsic::riscv_##NAME##NF: \
1122 return NF;
1123#define INST_NF_CASE_MASK(NAME, NF) \
1124 case Intrinsic::riscv_##NAME##NF##_mask: \
1125 return NF;
1126#define INST_NF_CASE_FF(NAME, NF) \
1127 case Intrinsic::riscv_##NAME##NF##ff: \
1128 return NF;
1129#define INST_NF_CASE_FF_MASK(NAME, NF) \
1130 case Intrinsic::riscv_##NAME##NF##ff_mask: \
1131 return NF;
1132#define INST_ALL_NF_CASE_BASE(MACRO_NAME, NAME) \
1133 MACRO_NAME(NAME, 2) \
1134 MACRO_NAME(NAME, 3) \
1135 MACRO_NAME(NAME, 4) \
1136 MACRO_NAME(NAME, 5) \
1137 MACRO_NAME(NAME, 6) \
1138 MACRO_NAME(NAME, 7) \
1139 MACRO_NAME(NAME, 8)
1140#define INST_ALL_NF_CASE(NAME) \
1141 INST_ALL_NF_CASE_BASE(INST_NF_CASE, NAME) \
1142 INST_ALL_NF_CASE_BASE(INST_NF_CASE_MASK, NAME)
1143#define INST_ALL_NF_CASE_WITH_FF(NAME) \
1144 INST_ALL_NF_CASE(NAME) \
1145 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF, NAME) \
1146 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF_MASK, NAME)
1147 switch (Intrinsic) {
1148 default:
1149 llvm_unreachable("Unexpected segment load/store intrinsic");
1150 INST_ALL_NF_CASE_WITH_FF(vlseg)
1151 INST_ALL_NF_CASE(vlsseg)
1152 INST_ALL_NF_CASE(vloxseg)
1153 INST_ALL_NF_CASE(vluxseg)
1154 INST_ALL_NF_CASE(vsseg)
1155 INST_ALL_NF_CASE(vssseg)
1156 INST_ALL_NF_CASE(vsoxseg)
1157 INST_ALL_NF_CASE(vsuxseg)
1158 }
1159}
1160
1161static bool isApplicableToPLIOrPLUI(int Val) {
1162 // Check if the immediate is packed i8 or i10
1163 int16_t Bit31To16 = Val >> 16;
1164 int16_t Bit15To0 = Val;
1165 int8_t Bit15To8 = Bit15To0 >> 8;
1166 int8_t Bit7To0 = Val;
1167 if (Bit31To16 != Bit15To0)
1168 return false;
1169
1170 return isInt<10>(x: Bit15To0) || isShiftedInt<10, 6>(x: Bit15To0) ||
1171 Bit15To8 == Bit7To0;
1172}
1173
1174void RISCVDAGToDAGISel::Select(SDNode *Node) {
1175 // If we have a custom node, we have already selected.
1176 if (Node->isMachineOpcode()) {
1177 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
1178 Node->setNodeId(-1);
1179 return;
1180 }
1181
1182 // Instruction Selection not handled by the auto-generated tablegen selection
1183 // should be handled here.
1184 unsigned Opcode = Node->getOpcode();
1185 MVT XLenVT = Subtarget->getXLenVT();
1186 SDLoc DL(Node);
1187 MVT VT = Node->getSimpleValueType(ResNo: 0);
1188
1189 bool HasBitTest = Subtarget->hasBEXTILike();
1190
1191 switch (Opcode) {
1192 case ISD::Constant: {
1193 assert(VT == Subtarget->getXLenVT() && "Unexpected VT");
1194 auto *ConstNode = cast<ConstantSDNode>(Val: Node);
1195 if (ConstNode->isZero()) {
1196 SDValue New =
1197 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: RISCV::X0, VT);
1198 ReplaceNode(F: Node, T: New.getNode());
1199 return;
1200 }
1201 int64_t Imm = ConstNode->getSExtValue();
1202 // If only the lower 8 bits are used, try to convert this to a simm6 by
1203 // sign-extending bit 7. This is neutral without the C extension, and
1204 // allows C.LI to be used if C is present.
1205 if (!isInt<8>(x: Imm) && isUInt<8>(x: Imm) && isInt<6>(x: SignExtend64<8>(x: Imm)) &&
1206 hasAllBUsers(Node))
1207 Imm = SignExtend64<8>(x: Imm);
1208 // If the upper XLen-16 bits are not used, try to convert this to a simm12
1209 // by sign extending bit 15.
1210 else if (!isInt<16>(x: Imm) && isUInt<16>(x: Imm) &&
1211 isInt<12>(x: SignExtend64<16>(x: Imm)) && hasAllHUsers(Node))
1212 Imm = SignExtend64<16>(x: Imm);
1213
1214 // If the upper XLen-16 bits are not used, the lower 2 bytes are the same,
1215 // and we can't use li, convert to an xlen splat so we can use pli.b.
1216 if (Subtarget->hasStdExtP() && !isInt<12>(x: Imm) &&
1217 (Imm & 0xff) == ((Imm >> 8) & 0xff) && hasAllHUsers(Node)) {
1218 // Splat the lower 16 bits to XLen. Sign extend for RV32.
1219 uint64_t Splat = Imm & 0xffff;
1220 Splat = (Splat << 16) | Splat;
1221 if (VT == MVT::i64)
1222 Imm = Splat << 32 | Splat;
1223 else
1224 Imm = SignExtend64<32>(x: Splat);
1225 } else {
1226 // If the upper 32-bits are not used try to convert this into a simm32 by
1227 // sign extending bit 32.
1228 if (!isInt<32>(x: Imm) && isUInt<32>(x: Imm) && hasAllWUsers(Node))
1229 Imm = SignExtend64<32>(x: Imm);
1230
1231 if (VT == MVT::i64 && !isInt<12>(x: Imm) && !isShiftedInt<20, 12>(x: Imm) &&
1232 Subtarget->hasStdExtP() && isApplicableToPLIOrPLUI(Val: Imm) &&
1233 hasAllWUsers(Node)) {
1234 // If it's 4 packed 8-bit integers or 2 packed signed 16-bit integers,
1235 // we can simply copy lower 32 bits to higher 32 bits to make it able to
1236 // rematerialize to PLI_B or PLI_H
1237 Imm = ((uint64_t)Imm << 32) | (Imm & 0xFFFFFFFF);
1238 }
1239 }
1240
1241 ReplaceNode(F: Node, T: selectImm(CurDAG, DL, VT, Imm, Subtarget: *Subtarget).getNode());
1242 return;
1243 }
1244 case ISD::ConstantFP: {
1245 const APFloat &APF = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1246
1247 bool Is64Bit = Subtarget->is64Bit();
1248 bool HasZdinx = Subtarget->hasStdExtZdinx();
1249
1250 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
1251 SDValue Imm;
1252 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
1253 // create an integer immediate.
1254 if (APF.isPosZero() || NegZeroF64) {
1255 if (VT == MVT::f64 && HasZdinx && !Is64Bit)
1256 Imm = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::f64);
1257 else
1258 Imm = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
1259 } else {
1260 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
1261 Subtarget: *Subtarget);
1262 }
1263
1264 unsigned Opc;
1265 switch (VT.SimpleTy) {
1266 default:
1267 llvm_unreachable("Unexpected size");
1268 case MVT::bf16:
1269 assert(Subtarget->hasStdExtZfbfmin());
1270 Opc = RISCV::FMV_H_X;
1271 break;
1272 case MVT::f16:
1273 Opc = Subtarget->hasStdExtZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
1274 break;
1275 case MVT::f32:
1276 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
1277 break;
1278 case MVT::f64:
1279 // For RV32, we can't move from a GPR, we need to convert instead. This
1280 // should only happen for +0.0 and -0.0.
1281 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
1282 if (HasZdinx)
1283 Opc = RISCV::COPY;
1284 else
1285 Opc = Is64Bit ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
1286 break;
1287 }
1288
1289 SDNode *Res;
1290 if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
1291 Res =
1292 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_16, DL, VT, Operand: Imm).getNode();
1293 } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
1294 Res =
1295 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_32, DL, VT, Operand: Imm).getNode();
1296 } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
1297 Res = CurDAG->getMachineNode(
1298 Opcode: Opc, dl: DL, VT, Op1: Imm,
1299 Op2: CurDAG->getTargetConstant(Val: RISCVFPRndMode::RNE, DL, VT: XLenVT));
1300 else
1301 Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: Imm);
1302
1303 // For f64 -0.0, we need to insert a fneg.d idiom.
1304 if (NegZeroF64) {
1305 Opc = RISCV::FSGNJN_D;
1306 if (HasZdinx)
1307 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
1308 Res =
1309 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: SDValue(Res, 0), Op2: SDValue(Res, 0));
1310 }
1311
1312 ReplaceNode(F: Node, T: Res);
1313 return;
1314 }
1315 case RISCVISD::BuildGPRPair:
1316 case RISCVISD::BuildPairF64:
1317 case RISCVISD::BuildPairGPRVec: {
1318 if (Opcode == RISCVISD::BuildPairF64 && !Subtarget->hasStdExtZdinx())
1319 break;
1320
1321 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::BuildPairF64) &&
1322 "BuildPairF64 only handled here on rv32i_zdinx");
1323
1324 SDValue N =
1325 buildGPRPair(CurDAG, DL, VT, Lo: Node->getOperand(Num: 0), Hi: Node->getOperand(Num: 1));
1326 ReplaceNode(F: Node, T: N.getNode());
1327 return;
1328 }
1329 case RISCVISD::SplitGPRPair:
1330 case RISCVISD::SplitF64:
1331 case RISCVISD::SplitGPRVec: {
1332 if (Subtarget->hasStdExtZdinx() || Opcode != RISCVISD::SplitF64) {
1333 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::SplitF64) &&
1334 "SplitF64 only handled here on rv32i_zdinx");
1335
1336 if (!SDValue(Node, 0).use_empty()) {
1337 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL,
1338 VT: Node->getValueType(ResNo: 0),
1339 Operand: Node->getOperand(Num: 0));
1340 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1341 }
1342
1343 if (!SDValue(Node, 1).use_empty()) {
1344 SDValue Hi = CurDAG->getTargetExtractSubreg(
1345 SRIdx: RISCV::sub_gpr_odd, DL, VT: Node->getValueType(ResNo: 1), Operand: Node->getOperand(Num: 0));
1346 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1347 }
1348
1349 CurDAG->RemoveDeadNode(N: Node);
1350 return;
1351 }
1352
1353 if (!Subtarget->hasStdExtZfa())
1354 break;
1355 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
1356 "Unexpected subtarget");
1357
1358 // With Zfa, lower to fmv.x.w and fmvh.x.d.
1359 if (!SDValue(Node, 0).use_empty()) {
1360 SDNode *Lo = CurDAG->getMachineNode(Opcode: RISCV::FMV_X_W_FPR64, dl: DL, VT,
1361 Op1: Node->getOperand(Num: 0));
1362 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Lo, 0));
1363 }
1364 if (!SDValue(Node, 1).use_empty()) {
1365 SDNode *Hi = CurDAG->getMachineNode(Opcode: RISCV::FMVH_X_D, dl: DL, VT,
1366 Op1: Node->getOperand(Num: 0));
1367 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Hi, 0));
1368 }
1369
1370 CurDAG->RemoveDeadNode(N: Node);
1371 return;
1372 }
1373 case ISD::SHL: {
1374 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1375 if (!N1C)
1376 break;
1377 SDValue N0 = Node->getOperand(Num: 0);
1378 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
1379 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1380 break;
1381 unsigned ShAmt = N1C->getZExtValue();
1382 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1383
1384 if (isShiftedMask_64(Value: Mask)) {
1385 unsigned XLen = Subtarget->getXLen();
1386 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1387 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1388 if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
1389 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
1390 // where C2 has 32 leading zeros and C3 trailing zeros.
1391 SDNode *SRLIW = CurDAG->getMachineNode(
1392 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1393 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1394 SDNode *SLLI = CurDAG->getMachineNode(
1395 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1396 Op2: CurDAG->getTargetConstant(Val: TrailingZeros + ShAmt, DL, VT));
1397 ReplaceNode(F: Node, T: SLLI);
1398 return;
1399 }
1400 if (TrailingZeros == 0 && LeadingZeros > ShAmt &&
1401 XLen - LeadingZeros > 11 && LeadingZeros != 32) {
1402 // Optimize (shl (and X, C2), C) -> (srli (slli X, C4), C4-C)
1403 // where C2 has C4 leading zeros and no trailing zeros.
1404 // This is profitable if the "and" was to be lowered to
1405 // (srli (slli X, C4), C4) and not (andi X, C2).
1406 // For "LeadingZeros == 32":
1407 // - with Zba it's just (slli.uw X, C)
1408 // - without Zba a tablegen pattern applies the very same
1409 // transform as we would have done here
1410 SDNode *SLLI = CurDAG->getMachineNode(
1411 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1412 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1413 SDNode *SRLI = CurDAG->getMachineNode(
1414 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1415 Op2: CurDAG->getTargetConstant(Val: LeadingZeros - ShAmt, DL, VT));
1416 ReplaceNode(F: Node, T: SRLI);
1417 return;
1418 }
1419 }
1420 break;
1421 }
1422 case ISD::SRL: {
1423 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1424 if (!N1C)
1425 break;
1426 SDValue N0 = Node->getOperand(Num: 0);
1427 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1428 break;
1429 unsigned ShAmt = N1C->getZExtValue();
1430 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1431
1432 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1433 // 32 leading zeros and C3 trailing zeros.
1434 if (isShiftedMask_64(Value: Mask) && N0.hasOneUse()) {
1435 unsigned XLen = Subtarget->getXLen();
1436 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1437 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1438 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1439 SDNode *SRLIW = CurDAG->getMachineNode(
1440 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1441 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1442 SDNode *SLLI = CurDAG->getMachineNode(
1443 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1444 Op2: CurDAG->getTargetConstant(Val: TrailingZeros - ShAmt, DL, VT));
1445 ReplaceNode(F: Node, T: SLLI);
1446 return;
1447 }
1448 }
1449
1450 // Optimize (srl (and X, C2), C) ->
1451 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1452 // Where C2 is a mask with C3 trailing ones.
1453 // Taking into account that the C2 may have had lower bits unset by
1454 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1455 // This pattern occurs when type legalizing right shifts for types with
1456 // less than XLen bits.
1457 Mask |= maskTrailingOnes<uint64_t>(N: ShAmt);
1458 if (!isMask_64(Value: Mask))
1459 break;
1460 unsigned TrailingOnes = llvm::countr_one(Value: Mask);
1461 if (ShAmt >= TrailingOnes)
1462 break;
1463 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1464 if (TrailingOnes == 32) {
1465 SDNode *SRLI = CurDAG->getMachineNode(
1466 Opcode: Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, dl: DL, VT,
1467 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1468 ReplaceNode(F: Node, T: SRLI);
1469 return;
1470 }
1471
1472 // Only do the remaining transforms if the AND has one use.
1473 if (!N0.hasOneUse())
1474 break;
1475
1476 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1477 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1478 SDNode *BEXTI = CurDAG->getMachineNode(
1479 Opcode: Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, dl: DL, VT,
1480 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1481 ReplaceNode(F: Node, T: BEXTI);
1482 return;
1483 }
1484
1485 const unsigned Msb = TrailingOnes - 1;
1486 const unsigned Lsb = ShAmt;
1487 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0.getOperand(i: 0), Msb, Lsb))
1488 return;
1489
1490 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1491 SDNode *SLLI =
1492 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1493 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1494 SDNode *SRLI = CurDAG->getMachineNode(
1495 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1496 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1497 ReplaceNode(F: Node, T: SRLI);
1498 return;
1499 }
1500 case ISD::SRA: {
1501 if (trySignedBitfieldExtract(Node))
1502 return;
1503
1504 if (trySignedBitfieldInsertInSign(Node))
1505 return;
1506
1507 // Optimize (sra (sext_inreg X, i16), C) ->
1508 // (srai (slli X, (XLen-16), (XLen-16) + C)
1509 // And (sra (sext_inreg X, i8), C) ->
1510 // (srai (slli X, (XLen-8), (XLen-8) + C)
1511 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1512 // This transform matches the code we get without Zbb. The shifts are more
1513 // compressible, and this can help expose CSE opportunities in the sdiv by
1514 // constant optimization.
1515 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1516 if (!N1C)
1517 break;
1518 SDValue N0 = Node->getOperand(Num: 0);
1519 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1520 break;
1521 unsigned ShAmt = N1C->getZExtValue();
1522 unsigned ExtSize =
1523 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
1524 // ExtSize of 32 should use sraiw via tablegen pattern.
1525 if (ExtSize >= 32 || ShAmt >= ExtSize)
1526 break;
1527 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1528 SDNode *SLLI =
1529 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1530 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1531 SDNode *SRAI = CurDAG->getMachineNode(
1532 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1533 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1534 ReplaceNode(F: Node, T: SRAI);
1535 return;
1536 }
1537 case ISD::SIGN_EXTEND_INREG: {
1538 // Optimize (sext_inreg (srl X, C), i8/i16) ->
1539 // (srai (slli X, XLen-ExtSize-C), XLen-ExtSize)
1540 // This is a bitfield extract pattern where we're extracting a signed
1541 // 8-bit or 16-bit field from position C.
1542 SDValue N0 = Node->getOperand(Num: 0);
1543 if (N0.getOpcode() != ISD::SRL || !N0.hasOneUse())
1544 break;
1545
1546 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1547 if (!ShAmtC)
1548 break;
1549
1550 unsigned ExtSize =
1551 cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT().getSizeInBits();
1552 unsigned ShAmt = ShAmtC->getZExtValue();
1553 unsigned XLen = Subtarget->getXLen();
1554
1555 // Only handle types less than 32, and make sure the shift amount is valid.
1556 if (ExtSize >= 32 || ShAmt >= XLen - ExtSize)
1557 break;
1558
1559 unsigned LShAmt = XLen - ExtSize - ShAmt;
1560 SDNode *SLLI =
1561 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1562 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1563 SDNode *SRAI = CurDAG->getMachineNode(
1564 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1565 Op2: CurDAG->getTargetConstant(Val: XLen - ExtSize, DL, VT));
1566 ReplaceNode(F: Node, T: SRAI);
1567 return;
1568 }
1569 case ISD::OR: {
1570 if (tryShrinkShlLogicImm(Node))
1571 return;
1572
1573 break;
1574 }
1575 case ISD::XOR:
1576 if (tryShrinkShlLogicImm(Node))
1577 return;
1578
1579 break;
1580 case ISD::AND: {
1581 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1582 if (!N1C)
1583 break;
1584
1585 SDValue N0 = Node->getOperand(Num: 0);
1586
1587 bool LeftShift = N0.getOpcode() == ISD::SHL;
1588 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1589 auto *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1590 if (!C)
1591 break;
1592 unsigned C2 = C->getZExtValue();
1593 unsigned XLen = Subtarget->getXLen();
1594 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1595
1596 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1597 // shift pair might offer more compression opportunities.
1598 // TODO: We could check for C extension here, but we don't have many lit
1599 // tests with the C extension enabled so not checking gets better
1600 // coverage.
1601 // TODO: What if ANDI faster than shift?
1602 bool IsCANDI = isInt<6>(x: N1C->getSExtValue());
1603
1604 uint64_t C1 = N1C->getZExtValue();
1605
1606 // Clear irrelevant bits in the mask.
1607 if (LeftShift)
1608 C1 &= maskTrailingZeros<uint64_t>(N: C2);
1609 else
1610 C1 &= maskTrailingOnes<uint64_t>(N: XLen - C2);
1611
1612 // Some transforms should only be done if the shift has a single use or
1613 // the AND would become (srli (slli X, 32), 32)
1614 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1615
1616 SDValue X = N0.getOperand(i: 0);
1617
1618 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1619 // with c3 leading zeros.
1620 if (!LeftShift && isMask_64(Value: C1)) {
1621 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1622 if (C2 < Leading) {
1623 // If the number of leading zeros is C2+32 this can be SRLIW.
1624 if (C2 + 32 == Leading) {
1625 SDNode *SRLIW = CurDAG->getMachineNode(
1626 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X, Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1627 ReplaceNode(F: Node, T: SRLIW);
1628 return;
1629 }
1630
1631 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1632 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1633 //
1634 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1635 // legalized and goes through DAG combine.
1636 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1637 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1638 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32) {
1639 SDNode *SRAIW =
1640 CurDAG->getMachineNode(Opcode: RISCV::SRAIW, dl: DL, VT, Op1: X.getOperand(i: 0),
1641 Op2: CurDAG->getTargetConstant(Val: 31, DL, VT));
1642 SDNode *SRLIW = CurDAG->getMachineNode(
1643 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: SDValue(SRAIW, 0),
1644 Op2: CurDAG->getTargetConstant(Val: Leading - 32, DL, VT));
1645 ReplaceNode(F: Node, T: SRLIW);
1646 return;
1647 }
1648
1649 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1650 // available.
1651 // Transform (and (srl x, C2), C1)
1652 // -> (<bfextract> x, msb, lsb)
1653 //
1654 // Make sure to keep this below the SRLIW cases, as we always want to
1655 // prefer the more common instruction.
1656 const unsigned Msb = llvm::bit_width(Value: C1) + C2 - 1;
1657 const unsigned Lsb = C2;
1658 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1659 return;
1660
1661 // (srli (slli x, c3-c2), c3).
1662 // Skip if we could use (zext.w (sraiw X, C2)).
1663 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1664 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1665 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32;
1666 // Also Skip if we can use bexti or th.tst.
1667 Skip |= HasBitTest && Leading == XLen - 1;
1668 if (OneUseOrZExtW && !Skip) {
1669 SDNode *SLLI = CurDAG->getMachineNode(
1670 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1671 Op2: CurDAG->getTargetConstant(Val: Leading - C2, DL, VT));
1672 SDNode *SRLI = CurDAG->getMachineNode(
1673 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1674 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1675 ReplaceNode(F: Node, T: SRLI);
1676 return;
1677 }
1678 }
1679 }
1680
1681 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1682 // shifted by c2 bits with c3 leading zeros.
1683 if (LeftShift && isShiftedMask_64(Value: C1)) {
1684 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1685
1686 if (C2 + Leading < XLen &&
1687 C1 == (maskTrailingOnes<uint64_t>(N: XLen - (C2 + Leading)) << C2)) {
1688 // Use slli.uw when possible.
1689 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1690 SDNode *SLLI_UW =
1691 CurDAG->getMachineNode(Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: X,
1692 Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1693 ReplaceNode(F: Node, T: SLLI_UW);
1694 return;
1695 }
1696
1697 // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
1698 // available.
1699 // Transform (and (shl x, c2), c1)
1700 // -> (<bfinsert> x, msb, lsb)
1701 // e.g.
1702 // (and (shl x, 12), 0x00fff000)
1703 // If XLen = 32 and C2 = 12, then
1704 // Msb = 32 - 8 - 1 = 23 and Lsb = 12
1705 const unsigned Msb = XLen - Leading - 1;
1706 const unsigned Lsb = C2;
1707 if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
1708 return;
1709
1710 if (OneUseOrZExtW && !IsCANDI) {
1711 // (packh x0, X)
1712 if (Subtarget->hasStdExtZbkb() && C1 == 0xff00 && C2 == 8) {
1713 SDNode *PACKH = CurDAG->getMachineNode(
1714 Opcode: RISCV::PACKH, dl: DL, VT,
1715 Op1: CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT()), Op2: X);
1716 ReplaceNode(F: Node, T: PACKH);
1717 return;
1718 }
1719 // (srli (slli c2+c3), c3)
1720 SDNode *SLLI = CurDAG->getMachineNode(
1721 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1722 Op2: CurDAG->getTargetConstant(Val: C2 + Leading, DL, VT));
1723 SDNode *SRLI = CurDAG->getMachineNode(
1724 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1725 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1726 ReplaceNode(F: Node, T: SRLI);
1727 return;
1728 }
1729 }
1730 }
1731
1732 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1733 // shifted mask with c2 leading zeros and c3 trailing zeros.
1734 if (!LeftShift && isShiftedMask_64(Value: C1)) {
1735 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1736 unsigned Trailing = llvm::countr_zero(Val: C1);
1737 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1738 !IsCANDI) {
1739 unsigned SrliOpc = RISCV::SRLI;
1740 // If the input is zexti32 we should use SRLIW.
1741 if (X.getOpcode() == ISD::AND &&
1742 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1743 X.getConstantOperandVal(i: 1) == UINT64_C(0xFFFFFFFF)) {
1744 SrliOpc = RISCV::SRLIW;
1745 X = X.getOperand(i: 0);
1746 }
1747 SDNode *SRLI = CurDAG->getMachineNode(
1748 Opcode: SrliOpc, dl: DL, VT, Op1: X,
1749 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1750 SDNode *SLLI = CurDAG->getMachineNode(
1751 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1752 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1753 ReplaceNode(F: Node, T: SLLI);
1754 return;
1755 }
1756 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1757 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1758 OneUseOrZExtW && !IsCANDI) {
1759 SDNode *SRLIW = CurDAG->getMachineNode(
1760 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1761 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1762 SDNode *SLLI = CurDAG->getMachineNode(
1763 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1764 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1765 ReplaceNode(F: Node, T: SLLI);
1766 return;
1767 }
1768 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1769 if (Trailing > 0 && Leading + Trailing == 32 && C2 + Trailing < XLen &&
1770 OneUseOrZExtW && Subtarget->hasStdExtZba()) {
1771 SDNode *SRLI = CurDAG->getMachineNode(
1772 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1773 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1774 SDNode *SLLI_UW = CurDAG->getMachineNode(
1775 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1776 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1777 ReplaceNode(F: Node, T: SLLI_UW);
1778 return;
1779 }
1780 }
1781
1782 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1783 // shifted mask with no leading zeros and c3 trailing zeros.
1784 if (LeftShift && isShiftedMask_64(Value: C1)) {
1785 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1786 unsigned Trailing = llvm::countr_zero(Val: C1);
1787 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1788 SDNode *SRLI = CurDAG->getMachineNode(
1789 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1790 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1791 SDNode *SLLI = CurDAG->getMachineNode(
1792 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1793 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1794 ReplaceNode(F: Node, T: SLLI);
1795 return;
1796 }
1797 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1798 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1799 SDNode *SRLIW = CurDAG->getMachineNode(
1800 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1801 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1802 SDNode *SLLI = CurDAG->getMachineNode(
1803 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1804 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1805 ReplaceNode(F: Node, T: SLLI);
1806 return;
1807 }
1808
1809 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1810 if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
1811 Subtarget->hasStdExtZba()) {
1812 SDNode *SRLI = CurDAG->getMachineNode(
1813 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1814 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1815 SDNode *SLLI_UW = CurDAG->getMachineNode(
1816 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1817 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1818 ReplaceNode(F: Node, T: SLLI_UW);
1819 return;
1820 }
1821 }
1822 }
1823
1824 const uint64_t C1 = N1C->getZExtValue();
1825
1826 if (N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
1827 N0.hasOneUse()) {
1828 unsigned C2 = N0.getConstantOperandVal(i: 1);
1829 unsigned XLen = Subtarget->getXLen();
1830 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1831
1832 SDValue X = N0.getOperand(i: 0);
1833
1834 // Prefer SRAIW + ANDI when possible.
1835 bool Skip = C2 > 32 && isInt<12>(x: N1C->getSExtValue()) &&
1836 X.getOpcode() == ISD::SHL &&
1837 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1838 X.getConstantOperandVal(i: 1) == 32;
1839 // Turn (and (sra x, c2), c1) -> (srli (srai x, c2-c3), c3) if c1 is a
1840 // mask with c3 leading zeros and c2 is larger than c3.
1841 if (isMask_64(Value: C1) && !Skip) {
1842 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1843 if (C2 > Leading) {
1844 SDNode *SRAI = CurDAG->getMachineNode(
1845 Opcode: RISCV::SRAI, dl: DL, VT, Op1: X,
1846 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1847 SDNode *SRLI = CurDAG->getMachineNode(
1848 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1849 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1850 ReplaceNode(F: Node, T: SRLI);
1851 return;
1852 }
1853 }
1854
1855 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
1856 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
1857 // use (slli (srli (srai y, c2 - c3), c3 + c4), c4).
1858 if (isShiftedMask_64(Value: C1) && !Skip) {
1859 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1860 unsigned Trailing = llvm::countr_zero(Val: C1);
1861 if (C2 > Leading && Leading > 0 && Trailing > 0) {
1862 SDNode *SRAI = CurDAG->getMachineNode(
1863 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1864 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1865 SDNode *SRLI = CurDAG->getMachineNode(
1866 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1867 Op2: CurDAG->getTargetConstant(Val: Leading + Trailing, DL, VT));
1868 SDNode *SLLI = CurDAG->getMachineNode(
1869 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1870 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1871 ReplaceNode(F: Node, T: SLLI);
1872 return;
1873 }
1874 }
1875 }
1876
1877 // If C1 masks off the upper bits only (but can't be formed as an
1878 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1879 // available.
1880 // Transform (and x, C1)
1881 // -> (<bfextract> x, msb, lsb)
1882 if (isMask_64(Value: C1) && !isInt<12>(x: N1C->getSExtValue()) &&
1883 !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
1884 !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
1885 const unsigned Msb = llvm::bit_width(Value: C1) - 1;
1886 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0, Msb, Lsb: 0))
1887 return;
1888 }
1889
1890 if (tryShrinkShlLogicImm(Node))
1891 return;
1892
1893 break;
1894 }
1895 case ISD::MUL: {
1896 // Special case for calculating (mul (and X, C2), C1) where the full product
1897 // fits in XLen bits. We can shift X left by the number of leading zeros in
1898 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1899 // product has XLen trailing zeros, putting it in the output of MULHU. This
1900 // can avoid materializing a constant in a register for C2.
1901
1902 // RHS should be a constant.
1903 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1904 if (!N1C || !N1C->hasOneUse())
1905 break;
1906
1907 // LHS should be an AND with constant.
1908 SDValue N0 = Node->getOperand(Num: 0);
1909 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1910 break;
1911
1912 uint64_t C2 = N0.getConstantOperandVal(i: 1);
1913
1914 // Constant should be a mask.
1915 if (!isMask_64(Value: C2))
1916 break;
1917
1918 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1919 // multiple users or the constant is a simm12. This prevents inserting a
1920 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1921 // make it more costly to materialize. Otherwise, using a SLLI might allow
1922 // it to be compressed.
1923 bool IsANDIOrZExt =
1924 isInt<12>(x: C2) ||
1925 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1926 // With XTHeadBb, we can use TH.EXTU.
1927 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1928 if (IsANDIOrZExt && (isInt<12>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1929 break;
1930 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1931 // the constant is a simm32.
1932 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1933 // With XTHeadBb, we can use TH.EXTU.
1934 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1935 if (IsZExtW && (isInt<32>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1936 break;
1937
1938 // We need to shift left the AND input and C1 by a total of XLen bits.
1939
1940 // How far left do we need to shift the AND input?
1941 unsigned XLen = Subtarget->getXLen();
1942 unsigned LeadingZeros = XLen - llvm::bit_width(Value: C2);
1943
1944 // The constant gets shifted by the remaining amount unless that would
1945 // shift bits out.
1946 uint64_t C1 = N1C->getZExtValue();
1947 unsigned ConstantShift = XLen - LeadingZeros;
1948 if (ConstantShift > (XLen - llvm::bit_width(Value: C1)))
1949 break;
1950
1951 uint64_t ShiftedC1 = C1 << ConstantShift;
1952 // If this RV32, we need to sign extend the constant.
1953 if (XLen == 32)
1954 ShiftedC1 = SignExtend64<32>(x: ShiftedC1);
1955
1956 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1957 SDNode *Imm = selectImm(CurDAG, DL, VT, Imm: ShiftedC1, Subtarget: *Subtarget).getNode();
1958 SDNode *SLLI =
1959 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1960 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1961 SDNode *MULHU = CurDAG->getMachineNode(Opcode: RISCV::MULHU, dl: DL, VT,
1962 Op1: SDValue(SLLI, 0), Op2: SDValue(Imm, 0));
1963 ReplaceNode(F: Node, T: MULHU);
1964 return;
1965 }
1966 case ISD::SMUL_LOHI:
1967 case ISD::UMUL_LOHI:
1968 case RISCVISD::WMULSU:
1969 case RISCVISD::WADD:
1970 case RISCVISD::WSUB:
1971 case RISCVISD::WADDU:
1972 case RISCVISD::WSUBU: {
1973 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1974 "Unexpected opcode");
1975
1976 unsigned Opc;
1977 switch (Node->getOpcode()) {
1978 default:
1979 llvm_unreachable("Unexpected opcode");
1980 case ISD::SMUL_LOHI:
1981 Opc = RISCV::WMUL;
1982 break;
1983 case ISD::UMUL_LOHI:
1984 Opc = RISCV::WMULU;
1985 break;
1986 case RISCVISD::WMULSU:
1987 Opc = RISCV::WMULSU;
1988 break;
1989 case RISCVISD::WADD:
1990 Opc = RISCV::WADD;
1991 break;
1992 case RISCVISD::WSUB:
1993 Opc = RISCV::WSUB;
1994 break;
1995 case RISCVISD::WADDU:
1996 Opc = RISCV::WADDU;
1997 break;
1998 case RISCVISD::WSUBU:
1999 Opc = RISCV::WSUBU;
2000 break;
2001 }
2002
2003 SDNode *Result = CurDAG->getMachineNode(
2004 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Node->getOperand(Num: 0), Op2: Node->getOperand(Num: 1));
2005
2006 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(Result, 0));
2007 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2008 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2009 CurDAG->RemoveDeadNode(N: Node);
2010 return;
2011 }
2012 case RISCVISD::WSLL:
2013 case RISCVISD::WSLA: {
2014 // Custom select WSLL/WSLA for RV32P.
2015 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
2016 "Unexpected opcode");
2017
2018 bool IsSigned = Node->getOpcode() == RISCVISD::WSLA;
2019
2020 SDValue ShAmt = Node->getOperand(Num: 1);
2021
2022 unsigned Opc;
2023
2024 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShAmt);
2025 if (ShAmtC && ShAmtC->getZExtValue() < 64) {
2026 Opc = IsSigned ? RISCV::WSLAI : RISCV::WSLLI;
2027 ShAmt = CurDAG->getTargetConstant(Val: ShAmtC->getZExtValue(), DL, VT: XLenVT);
2028 } else {
2029 Opc = IsSigned ? RISCV::WSLA : RISCV::WSLL;
2030 }
2031
2032 SDNode *WShift = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped,
2033 Op1: Node->getOperand(Num: 0), Op2: ShAmt);
2034
2035 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(WShift, 0));
2036 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2037 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2038 CurDAG->RemoveDeadNode(N: Node);
2039 return;
2040 }
2041 case ISD::LOAD: {
2042 if (tryIndexedLoad(Node))
2043 return;
2044
2045 if (Subtarget->hasVendorXCVmem() && !Subtarget->is64Bit()) {
2046 // We match post-incrementing load here
2047 LoadSDNode *Load = cast<LoadSDNode>(Val: Node);
2048 if (Load->getAddressingMode() != ISD::POST_INC)
2049 break;
2050
2051 SDValue Chain = Node->getOperand(Num: 0);
2052 SDValue Base = Node->getOperand(Num: 1);
2053 SDValue Offset = Node->getOperand(Num: 2);
2054
2055 bool Simm12 = false;
2056 bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
2057
2058 if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Val&: Offset)) {
2059 int ConstantVal = ConstantOffset->getSExtValue();
2060 Simm12 = isInt<12>(x: ConstantVal);
2061 if (Simm12)
2062 Offset = CurDAG->getSignedTargetConstant(Val: ConstantVal, DL: SDLoc(Offset),
2063 VT: Offset.getValueType());
2064 }
2065
2066 unsigned Opcode = 0;
2067 switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
2068 case MVT::i8:
2069 if (Simm12 && SignExtend)
2070 Opcode = RISCV::CV_LB_ri_inc;
2071 else if (Simm12 && !SignExtend)
2072 Opcode = RISCV::CV_LBU_ri_inc;
2073 else if (!Simm12 && SignExtend)
2074 Opcode = RISCV::CV_LB_rr_inc;
2075 else
2076 Opcode = RISCV::CV_LBU_rr_inc;
2077 break;
2078 case MVT::i16:
2079 if (Simm12 && SignExtend)
2080 Opcode = RISCV::CV_LH_ri_inc;
2081 else if (Simm12 && !SignExtend)
2082 Opcode = RISCV::CV_LHU_ri_inc;
2083 else if (!Simm12 && SignExtend)
2084 Opcode = RISCV::CV_LH_rr_inc;
2085 else
2086 Opcode = RISCV::CV_LHU_rr_inc;
2087 break;
2088 case MVT::i32:
2089 if (Simm12)
2090 Opcode = RISCV::CV_LW_ri_inc;
2091 else
2092 Opcode = RISCV::CV_LW_rr_inc;
2093 break;
2094 default:
2095 break;
2096 }
2097 if (!Opcode)
2098 break;
2099
2100 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode, dl: DL, VT1: XLenVT, VT2: XLenVT,
2101 VT3: Chain.getSimpleValueType(), Op1: Base,
2102 Op2: Offset, Op3: Chain));
2103 return;
2104 }
2105 break;
2106 }
2107 case RISCVISD::LD_RV32: {
2108 assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd");
2109
2110 SDValue Base, Offset;
2111 SDValue Chain = Node->getOperand(Num: 0);
2112 SDValue Addr = Node->getOperand(Num: 1);
2113 SelectAddrRegImm(Addr, Base, Offset);
2114
2115 SDValue Ops[] = {Base, Offset, Chain};
2116 MachineSDNode *New = CurDAG->getMachineNode(
2117 Opcode: RISCV::LD_RV32, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2118 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2119 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2120 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2121 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2122 ReplaceUses(F: SDValue(Node, 2), T: SDValue(New, 1));
2123 CurDAG->RemoveDeadNode(N: Node);
2124 return;
2125 }
2126 case RISCVISD::SD_RV32: {
2127 SDValue Base, Offset;
2128 SDValue Chain = Node->getOperand(Num: 0);
2129 SDValue Addr = Node->getOperand(Num: 3);
2130 SelectAddrRegImm(Addr, Base, Offset);
2131
2132 SDValue Lo = Node->getOperand(Num: 1);
2133 SDValue Hi = Node->getOperand(Num: 2);
2134
2135 SDValue RegPair;
2136 // Peephole to use X0_Pair for storing zero.
2137 if (isNullConstant(V: Lo) && isNullConstant(V: Hi)) {
2138 RegPair = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2139 } else {
2140 RegPair = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo, Hi);
2141 }
2142
2143 MachineSDNode *New = CurDAG->getMachineNode(Opcode: RISCV::SD_RV32, dl: DL, VT: MVT::Other,
2144 Ops: {RegPair, Base, Offset, Chain});
2145 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2146 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0));
2147 CurDAG->RemoveDeadNode(N: Node);
2148 return;
2149 }
2150 case RISCVISD::MQWACC:
2151 case RISCVISD::MQRWACC:
2152 case RISCVISD::WMACC:
2153 case RISCVISD::WMACCU:
2154 case RISCVISD::WMACCSU: {
2155 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2156 "Unexpected opcode");
2157
2158 SDValue Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Node->getOperand(Num: 0),
2159 Hi: Node->getOperand(Num: 1));
2160 unsigned Opc;
2161 switch (Opcode) {
2162 default:
2163 llvm_unreachable("Unexpected opcode");
2164 case RISCVISD::MQWACC:
2165 Opc = RISCV::MQWACC;
2166 break;
2167 case RISCVISD::MQRWACC:
2168 Opc = RISCV::MQRWACC;
2169 break;
2170 case RISCVISD::WMACC:
2171 Opc = RISCV::WMACC;
2172 break;
2173 case RISCVISD::WMACCU:
2174 Opc = RISCV::WMACCU;
2175 break;
2176 case RISCVISD::WMACCSU:
2177 Opc = RISCV::WMACCSU;
2178 break;
2179 }
2180 MachineSDNode *New = CurDAG->getMachineNode(
2181 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Node->getOperand(Num: 2), Op3: Node->getOperand(Num: 3));
2182 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2183 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2184 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2185 CurDAG->RemoveDeadNode(N: Node);
2186 return;
2187 }
2188 case RISCVISD::ADDD:
2189 // Try to match WMACC pattern: ADDD where one operand pair comes from a
2190 // widening multiply.
2191 if (tryWideningMulAcc(Node, DL))
2192 return;
2193
2194 // Fall through to regular ADDD selection.
2195 [[fallthrough]];
2196 case RISCVISD::SUBD:
2197 case RISCVISD::WADDAU:
2198 case RISCVISD::WSUBAU:
2199 case RISCVISD::WADDA:
2200 case RISCVISD::WSUBA: {
2201 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2202 "Unexpected opcode");
2203
2204 SDValue Op0Lo = Node->getOperand(Num: 0);
2205 SDValue Op0Hi = Node->getOperand(Num: 1);
2206
2207 SDValue Op0;
2208 if (isNullConstant(V: Op0Lo) && isNullConstant(V: Op0Hi)) {
2209 Op0 = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2210 } else {
2211 Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op0Lo, Hi: Op0Hi);
2212 }
2213
2214 SDValue Op1Lo = Node->getOperand(Num: 2);
2215 SDValue Op1Hi = Node->getOperand(Num: 3);
2216
2217 MachineSDNode *New;
2218 if (Opcode == RISCVISD::WADDAU || Opcode == RISCVISD::WSUBAU ||
2219 Opcode == RISCVISD::WADDA || Opcode == RISCVISD::WSUBA) {
2220 // Widening accumulate: Op0 is the accumulator (GPRPair), Op1Lo and Op1Hi
2221 // are the two 32-bit values.
2222 unsigned Opc;
2223 switch (Opcode) {
2224 default:
2225 llvm_unreachable("Unexpected opcode");
2226 case RISCVISD::WADDAU:
2227 Opc = RISCV::WADDAU;
2228 break;
2229 case RISCVISD::WSUBAU:
2230 Opc = RISCV::WSUBAU;
2231 break;
2232 case RISCVISD::WADDA:
2233 Opc = RISCV::WADDA;
2234 break;
2235 case RISCVISD::WSUBA:
2236 Opc = RISCV::WSUBA;
2237 break;
2238 }
2239 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1Lo, Op3: Op1Hi);
2240 } else {
2241 SDValue Op1 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op1Lo, Hi: Op1Hi);
2242
2243 unsigned Opc;
2244 switch (Opcode) {
2245 default:
2246 llvm_unreachable("Unexpected opcode");
2247 case RISCVISD::ADDD:
2248 Opc = RISCV::ADDD;
2249 break;
2250 case RISCVISD::SUBD:
2251 Opc = RISCV::SUBD;
2252 break;
2253 }
2254 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1);
2255 }
2256
2257 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2258 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2259 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2260 CurDAG->RemoveDeadNode(N: Node);
2261 return;
2262 }
2263 case ISD::INTRINSIC_WO_CHAIN: {
2264 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
2265 switch (IntNo) {
2266 // By default we do not custom select any intrinsic.
2267 default:
2268 break;
2269 case Intrinsic::riscv_vmsgeu:
2270 case Intrinsic::riscv_vmsge: {
2271 SDValue Src1 = Node->getOperand(Num: 1);
2272 SDValue Src2 = Node->getOperand(Num: 2);
2273 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
2274 bool IsCmpConstant = false;
2275 bool IsCmpMinimum = false;
2276 // Only custom select scalar second operand.
2277 if (Src2.getValueType() != XLenVT)
2278 break;
2279 // Small constants are handled with patterns.
2280 int64_t CVal = 0;
2281 MVT Src1VT = Src1.getSimpleValueType();
2282 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2283 IsCmpConstant = true;
2284 CVal = C->getSExtValue();
2285 if (CVal >= -15 && CVal <= 16) {
2286 if (!IsUnsigned || CVal != 0)
2287 break;
2288 IsCmpMinimum = true;
2289 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2290 numBits: Src1VT.getScalarSizeInBits())
2291 .getSExtValue()) {
2292 IsCmpMinimum = true;
2293 }
2294 }
2295 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode, VMSGTOpcode;
2296 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2297 default:
2298 llvm_unreachable("Unexpected LMUL!");
2299#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2300 case RISCVVType::lmulenum: \
2301 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2302 : RISCV::PseudoVMSLT_VX_##suffix; \
2303 VMSGTOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix \
2304 : RISCV::PseudoVMSGT_VX_##suffix; \
2305 break;
2306 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2307 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2308 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2309 CASE_VMSLT_OPCODES(LMUL_1, M1)
2310 CASE_VMSLT_OPCODES(LMUL_2, M2)
2311 CASE_VMSLT_OPCODES(LMUL_4, M4)
2312 CASE_VMSLT_OPCODES(LMUL_8, M8)
2313#undef CASE_VMSLT_OPCODES
2314 }
2315 // Mask operations use the LMUL from the mask type.
2316 switch (RISCVTargetLowering::getLMUL(VT)) {
2317 default:
2318 llvm_unreachable("Unexpected LMUL!");
2319#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix) \
2320 case RISCVVType::lmulenum: \
2321 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
2322 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix; \
2323 break;
2324 CASE_VMNAND_VMSET_OPCODES(LMUL_F8, B64)
2325 CASE_VMNAND_VMSET_OPCODES(LMUL_F4, B32)
2326 CASE_VMNAND_VMSET_OPCODES(LMUL_F2, B16)
2327 CASE_VMNAND_VMSET_OPCODES(LMUL_1, B8)
2328 CASE_VMNAND_VMSET_OPCODES(LMUL_2, B4)
2329 CASE_VMNAND_VMSET_OPCODES(LMUL_4, B2)
2330 CASE_VMNAND_VMSET_OPCODES(LMUL_8, B1)
2331#undef CASE_VMNAND_VMSET_OPCODES
2332 }
2333 SDValue SEW = CurDAG->getTargetConstant(
2334 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2335 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2336 SDValue VL;
2337 selectVLOp(N: Node->getOperand(Num: 3), VL);
2338
2339 // If vmsge(u) with minimum value, expand it to vmset.
2340 if (IsCmpMinimum) {
2341 ReplaceNode(F: Node,
2342 T: CurDAG->getMachineNode(Opcode: VMSetOpcode, dl: DL, VT, Op1: VL, Op2: MaskSEW));
2343 return;
2344 }
2345
2346 if (IsCmpConstant) {
2347 SDValue Imm =
2348 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2349
2350 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMSGTOpcode, dl: DL, VT,
2351 Ops: {Src1, Imm, VL, SEW}));
2352 return;
2353 }
2354
2355 // Expand to
2356 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
2357 SDValue Cmp = SDValue(
2358 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2359 0);
2360 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMNANDOpcode, dl: DL, VT,
2361 Ops: {Cmp, Cmp, VL, MaskSEW}));
2362 return;
2363 }
2364 case Intrinsic::riscv_vmsgeu_mask:
2365 case Intrinsic::riscv_vmsge_mask: {
2366 SDValue Src1 = Node->getOperand(Num: 2);
2367 SDValue Src2 = Node->getOperand(Num: 3);
2368 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
2369 bool IsCmpConstant = false;
2370 bool IsCmpMinimum = false;
2371 // Only custom select scalar second operand.
2372 if (Src2.getValueType() != XLenVT)
2373 break;
2374 // Small constants are handled with patterns.
2375 MVT Src1VT = Src1.getSimpleValueType();
2376 int64_t CVal = 0;
2377 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2378 IsCmpConstant = true;
2379 CVal = C->getSExtValue();
2380 if (CVal >= -15 && CVal <= 16) {
2381 if (!IsUnsigned || CVal != 0)
2382 break;
2383 IsCmpMinimum = true;
2384 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2385 numBits: Src1VT.getScalarSizeInBits())
2386 .getSExtValue()) {
2387 IsCmpMinimum = true;
2388 }
2389 }
2390 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
2391 VMOROpcode, VMSGTMaskOpcode;
2392 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2393 default:
2394 llvm_unreachable("Unexpected LMUL!");
2395#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2396 case RISCVVType::lmulenum: \
2397 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2398 : RISCV::PseudoVMSLT_VX_##suffix; \
2399 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
2400 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
2401 VMSGTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix##_MASK \
2402 : RISCV::PseudoVMSGT_VX_##suffix##_MASK; \
2403 break;
2404 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2405 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2406 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2407 CASE_VMSLT_OPCODES(LMUL_1, M1)
2408 CASE_VMSLT_OPCODES(LMUL_2, M2)
2409 CASE_VMSLT_OPCODES(LMUL_4, M4)
2410 CASE_VMSLT_OPCODES(LMUL_8, M8)
2411#undef CASE_VMSLT_OPCODES
2412 }
2413 // Mask operations use the LMUL from the mask type.
2414 switch (RISCVTargetLowering::getLMUL(VT)) {
2415 default:
2416 llvm_unreachable("Unexpected LMUL!");
2417#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
2418 case RISCVVType::lmulenum: \
2419 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
2420 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
2421 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
2422 break;
2423 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, B64)
2424 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, B32)
2425 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, B16)
2426 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, B8)
2427 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, B4)
2428 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, B2)
2429 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, B1)
2430#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
2431 }
2432 SDValue SEW = CurDAG->getTargetConstant(
2433 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2434 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2435 SDValue VL;
2436 selectVLOp(N: Node->getOperand(Num: 5), VL);
2437 SDValue MaskedOff = Node->getOperand(Num: 1);
2438 SDValue Mask = Node->getOperand(Num: 4);
2439
2440 // If vmsge(u) with minimum value, expand it to vmor mask, maskedoff.
2441 if (IsCmpMinimum) {
2442 // We don't need vmor if the MaskedOff and the Mask are the same
2443 // value.
2444 if (Mask == MaskedOff) {
2445 ReplaceUses(F: Node, T: Mask.getNode());
2446 return;
2447 }
2448 ReplaceNode(F: Node,
2449 T: CurDAG->getMachineNode(Opcode: VMOROpcode, dl: DL, VT,
2450 Ops: {Mask, MaskedOff, VL, MaskSEW}));
2451 return;
2452 }
2453
2454 // If the MaskedOff value and the Mask are the same value use
2455 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
2456 // This avoids needing to copy v0 to vd before starting the next sequence.
2457 if (Mask == MaskedOff) {
2458 SDValue Cmp = SDValue(
2459 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2460 0);
2461 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMANDNOpcode, dl: DL, VT,
2462 Ops: {Mask, Cmp, VL, MaskSEW}));
2463 return;
2464 }
2465
2466 SDValue PolicyOp =
2467 CurDAG->getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
2468
2469 if (IsCmpConstant) {
2470 SDValue Imm =
2471 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2472
2473 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
2474 Opcode: VMSGTMaskOpcode, dl: DL, VT,
2475 Ops: {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp}));
2476 return;
2477 }
2478
2479 // Otherwise use
2480 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
2481 // The result is mask undisturbed.
2482 // We use the same instructions to emulate mask agnostic behavior, because
2483 // the agnostic result can be either undisturbed or all 1.
2484 SDValue Cmp = SDValue(CurDAG->getMachineNode(Opcode: VMSLTMaskOpcode, dl: DL, VT,
2485 Ops: {MaskedOff, Src1, Src2, Mask,
2486 VL, SEW, PolicyOp}),
2487 0);
2488 // vmxor.mm vd, vd, v0 is used to update active value.
2489 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMXOROpcode, dl: DL, VT,
2490 Ops: {Cmp, Mask, VL, MaskSEW}));
2491 return;
2492 }
2493 case Intrinsic::riscv_vsetvli:
2494 case Intrinsic::riscv_vsetvlimax:
2495 return selectVSETVLI(Node);
2496 case Intrinsic::riscv_sf_vsettnt:
2497 case Intrinsic::riscv_sf_vsettm:
2498 case Intrinsic::riscv_sf_vsettk:
2499 return selectXSfmmVSET(Node);
2500 }
2501 break;
2502 }
2503 case ISD::INTRINSIC_W_CHAIN: {
2504 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2505 switch (IntNo) {
2506 // By default we do not custom select any intrinsic.
2507 default:
2508 break;
2509 case Intrinsic::riscv_vlseg2:
2510 case Intrinsic::riscv_vlseg3:
2511 case Intrinsic::riscv_vlseg4:
2512 case Intrinsic::riscv_vlseg5:
2513 case Intrinsic::riscv_vlseg6:
2514 case Intrinsic::riscv_vlseg7:
2515 case Intrinsic::riscv_vlseg8: {
2516 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2517 /*IsStrided*/ false);
2518 return;
2519 }
2520 case Intrinsic::riscv_vlseg2_mask:
2521 case Intrinsic::riscv_vlseg3_mask:
2522 case Intrinsic::riscv_vlseg4_mask:
2523 case Intrinsic::riscv_vlseg5_mask:
2524 case Intrinsic::riscv_vlseg6_mask:
2525 case Intrinsic::riscv_vlseg7_mask:
2526 case Intrinsic::riscv_vlseg8_mask: {
2527 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2528 /*IsStrided*/ false);
2529 return;
2530 }
2531 case Intrinsic::riscv_vlsseg2:
2532 case Intrinsic::riscv_vlsseg3:
2533 case Intrinsic::riscv_vlsseg4:
2534 case Intrinsic::riscv_vlsseg5:
2535 case Intrinsic::riscv_vlsseg6:
2536 case Intrinsic::riscv_vlsseg7:
2537 case Intrinsic::riscv_vlsseg8: {
2538 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2539 /*IsStrided*/ true);
2540 return;
2541 }
2542 case Intrinsic::riscv_vlsseg2_mask:
2543 case Intrinsic::riscv_vlsseg3_mask:
2544 case Intrinsic::riscv_vlsseg4_mask:
2545 case Intrinsic::riscv_vlsseg5_mask:
2546 case Intrinsic::riscv_vlsseg6_mask:
2547 case Intrinsic::riscv_vlsseg7_mask:
2548 case Intrinsic::riscv_vlsseg8_mask: {
2549 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2550 /*IsStrided*/ true);
2551 return;
2552 }
2553 case Intrinsic::riscv_vloxseg2:
2554 case Intrinsic::riscv_vloxseg3:
2555 case Intrinsic::riscv_vloxseg4:
2556 case Intrinsic::riscv_vloxseg5:
2557 case Intrinsic::riscv_vloxseg6:
2558 case Intrinsic::riscv_vloxseg7:
2559 case Intrinsic::riscv_vloxseg8:
2560 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2561 /*IsOrdered*/ true);
2562 return;
2563 case Intrinsic::riscv_vluxseg2:
2564 case Intrinsic::riscv_vluxseg3:
2565 case Intrinsic::riscv_vluxseg4:
2566 case Intrinsic::riscv_vluxseg5:
2567 case Intrinsic::riscv_vluxseg6:
2568 case Intrinsic::riscv_vluxseg7:
2569 case Intrinsic::riscv_vluxseg8:
2570 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2571 /*IsOrdered*/ false);
2572 return;
2573 case Intrinsic::riscv_vloxseg2_mask:
2574 case Intrinsic::riscv_vloxseg3_mask:
2575 case Intrinsic::riscv_vloxseg4_mask:
2576 case Intrinsic::riscv_vloxseg5_mask:
2577 case Intrinsic::riscv_vloxseg6_mask:
2578 case Intrinsic::riscv_vloxseg7_mask:
2579 case Intrinsic::riscv_vloxseg8_mask:
2580 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2581 /*IsOrdered*/ true);
2582 return;
2583 case Intrinsic::riscv_vluxseg2_mask:
2584 case Intrinsic::riscv_vluxseg3_mask:
2585 case Intrinsic::riscv_vluxseg4_mask:
2586 case Intrinsic::riscv_vluxseg5_mask:
2587 case Intrinsic::riscv_vluxseg6_mask:
2588 case Intrinsic::riscv_vluxseg7_mask:
2589 case Intrinsic::riscv_vluxseg8_mask:
2590 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2591 /*IsOrdered*/ false);
2592 return;
2593 case Intrinsic::riscv_vlseg8ff:
2594 case Intrinsic::riscv_vlseg7ff:
2595 case Intrinsic::riscv_vlseg6ff:
2596 case Intrinsic::riscv_vlseg5ff:
2597 case Intrinsic::riscv_vlseg4ff:
2598 case Intrinsic::riscv_vlseg3ff:
2599 case Intrinsic::riscv_vlseg2ff: {
2600 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false);
2601 return;
2602 }
2603 case Intrinsic::riscv_vlseg8ff_mask:
2604 case Intrinsic::riscv_vlseg7ff_mask:
2605 case Intrinsic::riscv_vlseg6ff_mask:
2606 case Intrinsic::riscv_vlseg5ff_mask:
2607 case Intrinsic::riscv_vlseg4ff_mask:
2608 case Intrinsic::riscv_vlseg3ff_mask:
2609 case Intrinsic::riscv_vlseg2ff_mask: {
2610 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true);
2611 return;
2612 }
2613 case Intrinsic::riscv_vloxei:
2614 case Intrinsic::riscv_vloxei_mask:
2615 case Intrinsic::riscv_vluxei:
2616 case Intrinsic::riscv_vluxei_mask: {
2617 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
2618 IntNo == Intrinsic::riscv_vluxei_mask;
2619 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
2620 IntNo == Intrinsic::riscv_vloxei_mask;
2621
2622 MVT VT = Node->getSimpleValueType(ResNo: 0);
2623 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2624
2625 unsigned CurOp = 2;
2626 SmallVector<SDValue, 8> Operands;
2627 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2628
2629 MVT IndexVT;
2630 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2631 /*IsStridedOrIndexed*/ true, Operands,
2632 /*IsLoad=*/true, IndexVT: &IndexVT);
2633
2634 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2635 "Element count mismatch");
2636
2637 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2638 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2639 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2640 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2641 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2642 "index values when XLEN=32");
2643 }
2644 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
2645 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
2646 IndexLMUL: static_cast<unsigned>(IndexLMUL));
2647 MachineSDNode *Load =
2648 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2649
2650 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2651
2652 ReplaceNode(F: Node, T: Load);
2653 return;
2654 }
2655 case Intrinsic::riscv_vlm:
2656 case Intrinsic::riscv_vle:
2657 case Intrinsic::riscv_vle_mask:
2658 case Intrinsic::riscv_vlse:
2659 case Intrinsic::riscv_vlse_mask: {
2660 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
2661 IntNo == Intrinsic::riscv_vlse_mask;
2662 bool IsStrided =
2663 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
2664
2665 MVT VT = Node->getSimpleValueType(ResNo: 0);
2666 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2667
2668 // The riscv_vlm intrinsic are always tail agnostic and no passthru
2669 // operand at the IR level. In pseudos, they have both policy and
2670 // passthru operand. The passthru operand is needed to track the
2671 // "tail undefined" state, and the policy is there just for
2672 // for consistency - it will always be "don't care" for the
2673 // unmasked form.
2674 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
2675 unsigned CurOp = 2;
2676 SmallVector<SDValue, 8> Operands;
2677 if (HasPassthruOperand)
2678 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2679 else {
2680 // We eagerly lower to implicit_def (instead of undef), as we
2681 // otherwise fail to select nodes such as: nxv1i1 = undef
2682 SDNode *Passthru =
2683 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
2684 Operands.push_back(Elt: SDValue(Passthru, 0));
2685 }
2686 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2687 Operands, /*IsLoad=*/true);
2688
2689 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2690 const RISCV::VLEPseudo *P =
2691 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
2692 LMUL: static_cast<unsigned>(LMUL));
2693 MachineSDNode *Load =
2694 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2695
2696 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2697
2698 ReplaceNode(F: Node, T: Load);
2699 return;
2700 }
2701 case Intrinsic::riscv_vleff:
2702 case Intrinsic::riscv_vleff_mask: {
2703 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
2704
2705 MVT VT = Node->getSimpleValueType(ResNo: 0);
2706 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2707
2708 unsigned CurOp = 2;
2709 SmallVector<SDValue, 7> Operands;
2710 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2711 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2712 /*IsStridedOrIndexed*/ false, Operands,
2713 /*IsLoad=*/true);
2714
2715 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2716 const RISCV::VLEPseudo *P =
2717 RISCV::getVLEPseudo(Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
2718 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2719 MachineSDNode *Load = CurDAG->getMachineNode(
2720 Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2721 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2722
2723 ReplaceNode(F: Node, T: Load);
2724 return;
2725 }
2726 case Intrinsic::riscv_nds_vln:
2727 case Intrinsic::riscv_nds_vln_mask:
2728 case Intrinsic::riscv_nds_vlnu:
2729 case Intrinsic::riscv_nds_vlnu_mask: {
2730 bool IsMasked = IntNo == Intrinsic::riscv_nds_vln_mask ||
2731 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2732 bool IsUnsigned = IntNo == Intrinsic::riscv_nds_vlnu ||
2733 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2734
2735 MVT VT = Node->getSimpleValueType(ResNo: 0);
2736 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2737 unsigned CurOp = 2;
2738 SmallVector<SDValue, 8> Operands;
2739
2740 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2741 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2742 /*IsStridedOrIndexed=*/false, Operands,
2743 /*IsLoad=*/true);
2744
2745 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2746 const RISCV::NDSVLNPseudo *P = RISCV::getNDSVLNPseudo(
2747 Masked: IsMasked, Unsigned: IsUnsigned, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2748 MachineSDNode *Load =
2749 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2750
2751 if (auto *MemOp = dyn_cast<MemSDNode>(Val: Node))
2752 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {MemOp->getMemOperand()});
2753
2754 ReplaceNode(F: Node, T: Load);
2755 return;
2756 }
2757 }
2758 break;
2759 }
2760 case ISD::INTRINSIC_VOID: {
2761 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2762 switch (IntNo) {
2763 case Intrinsic::riscv_vsseg2:
2764 case Intrinsic::riscv_vsseg3:
2765 case Intrinsic::riscv_vsseg4:
2766 case Intrinsic::riscv_vsseg5:
2767 case Intrinsic::riscv_vsseg6:
2768 case Intrinsic::riscv_vsseg7:
2769 case Intrinsic::riscv_vsseg8: {
2770 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2771 /*IsStrided*/ false);
2772 return;
2773 }
2774 case Intrinsic::riscv_vsseg2_mask:
2775 case Intrinsic::riscv_vsseg3_mask:
2776 case Intrinsic::riscv_vsseg4_mask:
2777 case Intrinsic::riscv_vsseg5_mask:
2778 case Intrinsic::riscv_vsseg6_mask:
2779 case Intrinsic::riscv_vsseg7_mask:
2780 case Intrinsic::riscv_vsseg8_mask: {
2781 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2782 /*IsStrided*/ false);
2783 return;
2784 }
2785 case Intrinsic::riscv_vssseg2:
2786 case Intrinsic::riscv_vssseg3:
2787 case Intrinsic::riscv_vssseg4:
2788 case Intrinsic::riscv_vssseg5:
2789 case Intrinsic::riscv_vssseg6:
2790 case Intrinsic::riscv_vssseg7:
2791 case Intrinsic::riscv_vssseg8: {
2792 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2793 /*IsStrided*/ true);
2794 return;
2795 }
2796 case Intrinsic::riscv_vssseg2_mask:
2797 case Intrinsic::riscv_vssseg3_mask:
2798 case Intrinsic::riscv_vssseg4_mask:
2799 case Intrinsic::riscv_vssseg5_mask:
2800 case Intrinsic::riscv_vssseg6_mask:
2801 case Intrinsic::riscv_vssseg7_mask:
2802 case Intrinsic::riscv_vssseg8_mask: {
2803 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2804 /*IsStrided*/ true);
2805 return;
2806 }
2807 case Intrinsic::riscv_vsoxseg2:
2808 case Intrinsic::riscv_vsoxseg3:
2809 case Intrinsic::riscv_vsoxseg4:
2810 case Intrinsic::riscv_vsoxseg5:
2811 case Intrinsic::riscv_vsoxseg6:
2812 case Intrinsic::riscv_vsoxseg7:
2813 case Intrinsic::riscv_vsoxseg8:
2814 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2815 /*IsOrdered*/ true);
2816 return;
2817 case Intrinsic::riscv_vsuxseg2:
2818 case Intrinsic::riscv_vsuxseg3:
2819 case Intrinsic::riscv_vsuxseg4:
2820 case Intrinsic::riscv_vsuxseg5:
2821 case Intrinsic::riscv_vsuxseg6:
2822 case Intrinsic::riscv_vsuxseg7:
2823 case Intrinsic::riscv_vsuxseg8:
2824 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2825 /*IsOrdered*/ false);
2826 return;
2827 case Intrinsic::riscv_vsoxseg2_mask:
2828 case Intrinsic::riscv_vsoxseg3_mask:
2829 case Intrinsic::riscv_vsoxseg4_mask:
2830 case Intrinsic::riscv_vsoxseg5_mask:
2831 case Intrinsic::riscv_vsoxseg6_mask:
2832 case Intrinsic::riscv_vsoxseg7_mask:
2833 case Intrinsic::riscv_vsoxseg8_mask:
2834 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2835 /*IsOrdered*/ true);
2836 return;
2837 case Intrinsic::riscv_vsuxseg2_mask:
2838 case Intrinsic::riscv_vsuxseg3_mask:
2839 case Intrinsic::riscv_vsuxseg4_mask:
2840 case Intrinsic::riscv_vsuxseg5_mask:
2841 case Intrinsic::riscv_vsuxseg6_mask:
2842 case Intrinsic::riscv_vsuxseg7_mask:
2843 case Intrinsic::riscv_vsuxseg8_mask:
2844 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2845 /*IsOrdered*/ false);
2846 return;
2847 case Intrinsic::riscv_vsoxei:
2848 case Intrinsic::riscv_vsoxei_mask:
2849 case Intrinsic::riscv_vsuxei:
2850 case Intrinsic::riscv_vsuxei_mask: {
2851 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
2852 IntNo == Intrinsic::riscv_vsuxei_mask;
2853 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
2854 IntNo == Intrinsic::riscv_vsoxei_mask;
2855
2856 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2857 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2858
2859 unsigned CurOp = 2;
2860 SmallVector<SDValue, 8> Operands;
2861 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2862
2863 MVT IndexVT;
2864 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2865 /*IsStridedOrIndexed*/ true, Operands,
2866 /*IsLoad=*/false, IndexVT: &IndexVT);
2867
2868 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2869 "Element count mismatch");
2870
2871 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2872 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2873 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2874 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2875 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2876 "index values when XLEN=32");
2877 }
2878 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
2879 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW,
2880 LMUL: static_cast<unsigned>(LMUL), IndexLMUL: static_cast<unsigned>(IndexLMUL));
2881 MachineSDNode *Store =
2882 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2883
2884 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2885
2886 ReplaceNode(F: Node, T: Store);
2887 return;
2888 }
2889 case Intrinsic::riscv_vsm:
2890 case Intrinsic::riscv_vse:
2891 case Intrinsic::riscv_vse_mask:
2892 case Intrinsic::riscv_vsse:
2893 case Intrinsic::riscv_vsse_mask: {
2894 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
2895 IntNo == Intrinsic::riscv_vsse_mask;
2896 bool IsStrided =
2897 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
2898
2899 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2900 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2901
2902 unsigned CurOp = 2;
2903 SmallVector<SDValue, 8> Operands;
2904 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2905
2906 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2907 Operands);
2908
2909 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2910 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
2911 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2912 MachineSDNode *Store =
2913 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2914 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2915
2916 ReplaceNode(F: Node, T: Store);
2917 return;
2918 }
2919 case Intrinsic::riscv_sf_vc_x_se:
2920 case Intrinsic::riscv_sf_vc_i_se:
2921 selectSF_VC_X_SE(Node);
2922 return;
2923 case Intrinsic::riscv_sf_vlte8:
2924 case Intrinsic::riscv_sf_vlte16:
2925 case Intrinsic::riscv_sf_vlte32:
2926 case Intrinsic::riscv_sf_vlte64: {
2927 unsigned Log2SEW;
2928 unsigned PseudoInst;
2929 switch (IntNo) {
2930 case Intrinsic::riscv_sf_vlte8:
2931 PseudoInst = RISCV::PseudoSF_VLTE8;
2932 Log2SEW = 3;
2933 break;
2934 case Intrinsic::riscv_sf_vlte16:
2935 PseudoInst = RISCV::PseudoSF_VLTE16;
2936 Log2SEW = 4;
2937 break;
2938 case Intrinsic::riscv_sf_vlte32:
2939 PseudoInst = RISCV::PseudoSF_VLTE32;
2940 Log2SEW = 5;
2941 break;
2942 case Intrinsic::riscv_sf_vlte64:
2943 PseudoInst = RISCV::PseudoSF_VLTE64;
2944 Log2SEW = 6;
2945 break;
2946 }
2947
2948 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
2949 SDValue TWidenOp = CurDAG->getTargetConstant(Val: 1, DL, VT: XLenVT);
2950 SDValue Operands[] = {Node->getOperand(Num: 2),
2951 Node->getOperand(Num: 3),
2952 Node->getOperand(Num: 4),
2953 SEWOp,
2954 TWidenOp,
2955 Node->getOperand(Num: 0)};
2956
2957 MachineSDNode *TileLoad =
2958 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2959 CurDAG->setNodeMemRefs(N: TileLoad,
2960 NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2961
2962 ReplaceNode(F: Node, T: TileLoad);
2963 return;
2964 }
2965 case Intrinsic::riscv_sf_mm_s_s:
2966 case Intrinsic::riscv_sf_mm_s_u:
2967 case Intrinsic::riscv_sf_mm_u_s:
2968 case Intrinsic::riscv_sf_mm_u_u:
2969 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2970 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2971 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2972 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
2973 case Intrinsic::riscv_sf_mm_f_f: {
2974 bool HasFRM = false;
2975 unsigned PseudoInst;
2976 switch (IntNo) {
2977 case Intrinsic::riscv_sf_mm_s_s:
2978 PseudoInst = RISCV::PseudoSF_MM_S_S;
2979 break;
2980 case Intrinsic::riscv_sf_mm_s_u:
2981 PseudoInst = RISCV::PseudoSF_MM_S_U;
2982 break;
2983 case Intrinsic::riscv_sf_mm_u_s:
2984 PseudoInst = RISCV::PseudoSF_MM_U_S;
2985 break;
2986 case Intrinsic::riscv_sf_mm_u_u:
2987 PseudoInst = RISCV::PseudoSF_MM_U_U;
2988 break;
2989 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2990 PseudoInst = RISCV::PseudoSF_MM_E5M2_E5M2;
2991 HasFRM = true;
2992 break;
2993 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2994 PseudoInst = RISCV::PseudoSF_MM_E5M2_E4M3;
2995 HasFRM = true;
2996 break;
2997 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2998 PseudoInst = RISCV::PseudoSF_MM_E4M3_E5M2;
2999 HasFRM = true;
3000 break;
3001 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
3002 PseudoInst = RISCV::PseudoSF_MM_E4M3_E4M3;
3003 HasFRM = true;
3004 break;
3005 case Intrinsic::riscv_sf_mm_f_f:
3006 if (Node->getOperand(Num: 3).getValueType().getScalarType() == MVT::bf16)
3007 PseudoInst = RISCV::PseudoSF_MM_F_F_ALT;
3008 else
3009 PseudoInst = RISCV::PseudoSF_MM_F_F;
3010 HasFRM = true;
3011 break;
3012 }
3013 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3014 SDValue Op1 = Node->getOperand(Num: 3);
3015 SDValue Op2 = Node->getOperand(Num: 4);
3016 MVT VT = Op1->getSimpleValueType(ResNo: 0);
3017 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3018 SDValue TmOp = Node->getOperand(Num: 5);
3019 SDValue TnOp = Node->getOperand(Num: 6);
3020 SDValue TkOp = Node->getOperand(Num: 7);
3021 SDValue TWidenOp = Node->getOperand(Num: 8);
3022 SDValue Chain = Node->getOperand(Num: 0);
3023
3024 // sf.mm.f.f with sew=32, twiden=2 is invalid
3025 if (IntNo == Intrinsic::riscv_sf_mm_f_f && Log2SEW == 5 &&
3026 TWidenOp->getAsZExtVal() == 2)
3027 reportFatalUsageError(reason: "sf.mm.f.f doesn't support (sew=32, twiden=2)");
3028
3029 SmallVector<SDValue, 10> Operands(
3030 {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Op1, Op2});
3031 if (HasFRM)
3032 Operands.push_back(
3033 Elt: CurDAG->getTargetConstant(Val: RISCVFPRndMode::DYN, DL, VT: XLenVT));
3034 Operands.append(IL: {TmOp, TnOp, TkOp,
3035 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT), TWidenOp,
3036 Chain});
3037
3038 auto *NewNode =
3039 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
3040
3041 ReplaceNode(F: Node, T: NewNode);
3042 return;
3043 }
3044 case Intrinsic::riscv_sf_vtzero_t: {
3045 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3046 SDValue Tm = Node->getOperand(Num: 3);
3047 SDValue Tn = Node->getOperand(Num: 4);
3048 SDValue Log2SEW = Node->getOperand(Num: 5);
3049 SDValue TWiden = Node->getOperand(Num: 6);
3050 SDValue Chain = Node->getOperand(Num: 0);
3051 auto *NewNode = CurDAG->getMachineNode(
3052 Opcode: RISCV::PseudoSF_VTZERO_T, dl: DL, VTs: Node->getVTList(),
3053 Ops: {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Tm, Tn, Log2SEW,
3054 TWiden, Chain});
3055
3056 ReplaceNode(F: Node, T: NewNode);
3057 return;
3058 }
3059 }
3060 break;
3061 }
3062 case ISD::BITCAST: {
3063 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3064 // Just drop bitcasts between vectors if both are fixed or both are
3065 // scalable.
3066 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
3067 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
3068 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3069 CurDAG->RemoveDeadNode(N: Node);
3070 return;
3071 }
3072 if (Subtarget->hasStdExtP()) {
3073 bool Is32BitCast =
3074 (VT == MVT::i32 && (SrcVT == MVT::v4i8 || SrcVT == MVT::v2i16)) ||
3075 (SrcVT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
3076 bool Is64BitCast =
3077 (VT == MVT::i64 && (SrcVT == MVT::v8i8 || SrcVT == MVT::v4i16 ||
3078 SrcVT == MVT::v2i32)) ||
3079 (SrcVT == MVT::i64 &&
3080 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
3081 if (Is32BitCast || Is64BitCast) {
3082 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3083 CurDAG->RemoveDeadNode(N: Node);
3084 return;
3085 }
3086 }
3087 break;
3088 }
3089 case ISD::SPLAT_VECTOR: {
3090 if (!Subtarget->hasStdExtP())
3091 break;
3092 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0))) {
3093 bool IsDoubleWide = Subtarget->isPExtPackedDoubleType(VT);
3094
3095 if (ConstNode->isZero()) {
3096 MCPhysReg X0Reg = IsDoubleWide ? RISCV::X0_Pair : RISCV::X0;
3097 SDValue New =
3098 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: X0Reg, VT);
3099 ReplaceNode(F: Node, T: New.getNode());
3100 return;
3101 }
3102
3103 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3104 APInt Val = ConstNode->getAPIntValue().trunc(width: EltSize);
3105
3106 // Use LI for all ones since it can be compressed to c.li.
3107 if (Val.isAllOnes() && !IsDoubleWide) {
3108 SDNode *NewNode = CurDAG->getMachineNode(
3109 Opcode: RISCV::ADDI, dl: DL, VT, Op1: CurDAG->getRegister(Reg: RISCV::X0, VT),
3110 Op2: CurDAG->getAllOnesConstant(DL, VT: XLenVT, /*IsTarget=*/true));
3111 ReplaceNode(F: Node, T: NewNode);
3112 return;
3113 }
3114
3115 // Find the smallest splat.
3116 if (Val.getBitWidth() > 16 && Val.isSplat(SplatSizeInBits: 16))
3117 Val = Val.trunc(width: 16);
3118 if (Val.getBitWidth() > 8 && Val.isSplat(SplatSizeInBits: 8))
3119 Val = Val.trunc(width: 8);
3120
3121 EltSize = Val.getBitWidth();
3122 int64_t Imm = Val.getSExtValue();
3123
3124 unsigned Opc = 0;
3125 if (EltSize == 8) {
3126 Opc = IsDoubleWide ? RISCV::PLI_DB : RISCV::PLI_B;
3127 } else if (EltSize == 16 && isInt<10>(x: Imm)) {
3128 Opc = IsDoubleWide ? RISCV::PLI_DH : RISCV::PLI_H;
3129 } else if (!IsDoubleWide && EltSize == 32 && isInt<10>(x: Imm)) {
3130 Opc = RISCV::PLI_W;
3131 } else if (EltSize == 16 && isShiftedInt<10, 6>(x: Imm)) {
3132 Opc = IsDoubleWide ? RISCV::PLUI_DH : RISCV::PLUI_H;
3133 Imm = Imm >> 6;
3134 } else if (!IsDoubleWide && EltSize == 32 && isShiftedInt<10, 22>(x: Imm)) {
3135 Opc = RISCV::PLUI_W;
3136 Imm = Imm >> 22;
3137 }
3138
3139 if (Opc) {
3140 SDNode *NewNode = CurDAG->getMachineNode(
3141 Opcode: Opc, dl: DL, VT, Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT: XLenVT));
3142 ReplaceNode(F: Node, T: NewNode);
3143 return;
3144 }
3145 }
3146
3147 break;
3148 }
3149 case ISD::SCALAR_TO_VECTOR:
3150 if (Subtarget->hasStdExtP()) {
3151 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3152 if ((VT == MVT::v2i32 && SrcVT == MVT::i64) ||
3153 (VT == MVT::v4i8 && SrcVT == MVT::i32)) {
3154 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3155 CurDAG->RemoveDeadNode(N: Node);
3156 return;
3157 }
3158 }
3159 break;
3160 case ISD::INSERT_SUBVECTOR:
3161 case RISCVISD::TUPLE_INSERT: {
3162 SDValue V = Node->getOperand(Num: 0);
3163 SDValue SubV = Node->getOperand(Num: 1);
3164 SDLoc DL(SubV);
3165 auto Idx = Node->getConstantOperandVal(Num: 2);
3166 MVT SubVecVT = SubV.getSimpleValueType();
3167
3168 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3169 MVT SubVecContainerVT = SubVecVT;
3170 // Establish the correct scalable-vector types for any fixed-length type.
3171 if (SubVecVT.isFixedLengthVector()) {
3172 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT: SubVecVT);
3173 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
3174 [[maybe_unused]] bool ExactlyVecRegSized =
3175 Subtarget->expandVScale(X: SubVecVT.getSizeInBits())
3176 .isKnownMultipleOf(RHS: Subtarget->expandVScale(X: VecRegSize));
3177 assert(isPowerOf2_64(Subtarget->expandVScale(SubVecVT.getSizeInBits())
3178 .getKnownMinValue()));
3179 assert(Idx == 0 && (ExactlyVecRegSized || V.isUndef()));
3180 }
3181 MVT ContainerVT = VT;
3182 if (VT.isFixedLengthVector())
3183 ContainerVT = TLI.getContainerForFixedLengthVector(VT);
3184
3185 const auto *TRI = Subtarget->getRegisterInfo();
3186 unsigned SubRegIdx;
3187 std::tie(args&: SubRegIdx, args&: Idx) =
3188 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3189 VecVT: ContainerVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3190
3191 // If the Idx hasn't been completely eliminated then this is a subvector
3192 // insert which doesn't naturally align to a vector register. These must
3193 // be handled using instructions to manipulate the vector registers.
3194 if (Idx != 0)
3195 break;
3196
3197 RISCVVType::VLMUL SubVecLMUL =
3198 RISCVTargetLowering::getLMUL(VT: SubVecContainerVT);
3199 [[maybe_unused]] bool IsSubVecPartReg =
3200 SubVecLMUL == RISCVVType::VLMUL::LMUL_F2 ||
3201 SubVecLMUL == RISCVVType::VLMUL::LMUL_F4 ||
3202 SubVecLMUL == RISCVVType::VLMUL::LMUL_F8;
3203 assert((V.getValueType().isRISCVVectorTuple() || !IsSubVecPartReg ||
3204 V.isUndef()) &&
3205 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
3206 "the subvector is smaller than a full-sized register");
3207
3208 // If we haven't set a SubRegIdx, then we must be going between
3209 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
3210 if (SubRegIdx == RISCV::NoSubRegister) {
3211 unsigned InRegClassID =
3212 RISCVTargetLowering::getRegClassIDForVecVT(VT: ContainerVT);
3213 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3214 InRegClassID &&
3215 "Unexpected subvector extraction");
3216 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3217 SDNode *NewNode = CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
3218 dl: DL, VT, Op1: SubV, Op2: RC);
3219 ReplaceNode(F: Node, T: NewNode);
3220 return;
3221 }
3222
3223 SDValue Insert = CurDAG->getTargetInsertSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V, Subreg: SubV);
3224 ReplaceNode(F: Node, T: Insert.getNode());
3225 return;
3226 }
3227 case ISD::EXTRACT_SUBVECTOR:
3228 case RISCVISD::TUPLE_EXTRACT: {
3229 if (Subtarget->hasStdExtP())
3230 break;
3231
3232 SDValue V = Node->getOperand(Num: 0);
3233 auto Idx = Node->getConstantOperandVal(Num: 1);
3234 MVT InVT = V.getSimpleValueType();
3235
3236 SDLoc DL(V);
3237
3238 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3239 MVT SubVecContainerVT = VT;
3240 // Establish the correct scalable-vector types for any fixed-length type.
3241 if (VT.isFixedLengthVector()) {
3242 assert(Idx == 0);
3243 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
3244 }
3245 if (InVT.isFixedLengthVector())
3246 InVT = TLI.getContainerForFixedLengthVector(VT: InVT);
3247
3248 const auto *TRI = Subtarget->getRegisterInfo();
3249 unsigned SubRegIdx;
3250 std::tie(args&: SubRegIdx, args&: Idx) =
3251 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3252 VecVT: InVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3253
3254 // If the Idx hasn't been completely eliminated then this is a subvector
3255 // extract which doesn't naturally align to a vector register. These must
3256 // be handled using instructions to manipulate the vector registers.
3257 if (Idx != 0)
3258 break;
3259
3260 // If we haven't set a SubRegIdx, then we must be going between
3261 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
3262 if (SubRegIdx == RISCV::NoSubRegister) {
3263 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: InVT);
3264 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3265 InRegClassID &&
3266 "Unexpected subvector extraction");
3267 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3268 SDNode *NewNode =
3269 CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT, Op1: V, Op2: RC);
3270 ReplaceNode(F: Node, T: NewNode);
3271 return;
3272 }
3273
3274 SDValue Extract = CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V);
3275 ReplaceNode(F: Node, T: Extract.getNode());
3276 return;
3277 }
3278 case RISCVISD::VMV_S_X_VL:
3279 case RISCVISD::VFMV_S_F_VL:
3280 case RISCVISD::VMV_V_X_VL:
3281 case RISCVISD::VFMV_V_F_VL: {
3282 // Try to match splat of a scalar load to a strided load with stride of x0.
3283 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
3284 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
3285 if (!Node->getOperand(Num: 0).isUndef())
3286 break;
3287 SDValue Src = Node->getOperand(Num: 1);
3288 auto *Ld = dyn_cast<LoadSDNode>(Val&: Src);
3289 // Can't fold load update node because the second
3290 // output is used so that load update node can't be removed.
3291 if (!Ld || Ld->isIndexed())
3292 break;
3293 EVT MemVT = Ld->getMemoryVT();
3294 // The memory VT should be the same size as the element type.
3295 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
3296 break;
3297 if (!IsProfitableToFold(N: Src, U: Node, Root: Node) ||
3298 !IsLegalToFold(N: Src, U: Node, Root: Node, OptLevel: TM.getOptLevel()))
3299 break;
3300
3301 SDValue VL;
3302 if (IsScalarMove) {
3303 // We could deal with more VL if we update the VSETVLI insert pass to
3304 // avoid introducing more VSETVLI.
3305 if (!isOneConstant(V: Node->getOperand(Num: 2)))
3306 break;
3307 selectVLOp(N: Node->getOperand(Num: 2), VL);
3308 } else
3309 selectVLOp(N: Node->getOperand(Num: 2), VL);
3310
3311 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3312 SDValue SEW = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
3313
3314 // If VL=1, then we don't need to do a strided load and can just do a
3315 // regular load.
3316 bool IsStrided = !isOneConstant(V: VL);
3317
3318 // Only do a strided load if we have optimized zero-stride vector load.
3319 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
3320 break;
3321
3322 SmallVector<SDValue> Operands = {
3323 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT), 0),
3324 Ld->getBasePtr()};
3325 if (IsStrided)
3326 Operands.push_back(Elt: CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT));
3327 uint64_t Policy = RISCVVType::MASK_AGNOSTIC | RISCVVType::TAIL_AGNOSTIC;
3328 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
3329 Operands.append(IL: {VL, SEW, PolicyOp, Ld->getChain()});
3330
3331 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
3332 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
3333 /*IsMasked*/ Masked: false, Strided: IsStrided, /*FF*/ false,
3334 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
3335 MachineSDNode *Load =
3336 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, ResultTys: {VT, MVT::Other}, Ops: Operands);
3337 // Update the chain.
3338 ReplaceUses(F: Src.getValue(R: 1), T: SDValue(Load, 1));
3339 // Record the mem-refs
3340 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {Ld->getMemOperand()});
3341 // Replace the splat with the vlse.
3342 ReplaceNode(F: Node, T: Load);
3343 return;
3344 }
3345 case RISCVISD::LPAD_CALL:
3346 case RISCVISD::LPAD_CALL_INDIRECT: {
3347 bool IsIndirect = Opcode == RISCVISD::LPAD_CALL_INDIRECT;
3348 unsigned PseudoOpc = IsIndirect ? RISCV::PseudoCALLIndirectLpadAlign
3349 : RISCV::PseudoCALLLpadAlign;
3350
3351 uint32_t LpadLabel = 0;
3352 if (PreferredLandingPadLabel.getNumOccurrences() > 0) {
3353 if (!isUInt<20>(x: PreferredLandingPadLabel))
3354 report_fatal_error(reason: "riscv-landing-pad-label=<val>, <val> needs to fit "
3355 "in unsigned 20-bits");
3356 LpadLabel = PreferredLandingPadLabel;
3357 }
3358
3359 // Preserve the argument-register and register-mask operands, between
3360 // Callee and the optional glue, so the pseudo call still reports its
3361 // call-preserved mask to the register allocator.
3362 SmallVector<SDValue, 8> Ops;
3363 Ops.push_back(Elt: Node->getOperand(Num: 1));
3364 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: LpadLabel, DL, VT: XLenVT));
3365
3366 unsigned NumOps = Node->getNumOperands();
3367 bool HasGlue = Node->getGluedNode() != nullptr;
3368 unsigned RegOperandsEnd = HasGlue ? NumOps - 1 : NumOps;
3369 for (unsigned I = 2; I != RegOperandsEnd; ++I)
3370 Ops.push_back(Elt: Node->getOperand(Num: I));
3371
3372 Ops.push_back(Elt: Node->getOperand(Num: 0));
3373 if (HasGlue)
3374 Ops.push_back(Elt: Node->getOperand(Num: NumOps - 1));
3375
3376 ReplaceNode(F: Node,
3377 T: CurDAG->getMachineNode(Opcode: PseudoOpc, dl: DL, VTs: Node->getVTList(), Ops));
3378 return;
3379 }
3380 case ISD::PREFETCH:
3381 // MIPS's prefetch instruction already encodes the hint within the
3382 // instruction itself, so no extra NTL hint is needed.
3383 if (Subtarget->hasVendorXMIPSCBOP())
3384 break;
3385
3386 unsigned Locality = Node->getConstantOperandVal(Num: 3);
3387 if (Locality > 2)
3388 break;
3389
3390 auto *LoadStoreMem = cast<MemSDNode>(Val: Node);
3391 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
3392 MMO->setFlags(MachineMemOperand::MONonTemporal);
3393
3394 int NontemporalLevel = 0;
3395 switch (Locality) {
3396 case 0:
3397 NontemporalLevel = 3; // NTL.ALL
3398 break;
3399 case 1:
3400 NontemporalLevel = 1; // NTL.PALL
3401 break;
3402 case 2:
3403 NontemporalLevel = 0; // NTL.P1
3404 break;
3405 default:
3406 llvm_unreachable("unexpected locality value.");
3407 }
3408
3409 if (NontemporalLevel & 0b1)
3410 MMO->setFlags(MONontemporalBit0);
3411 if (NontemporalLevel & 0b10)
3412 MMO->setFlags(MONontemporalBit1);
3413 break;
3414 }
3415
3416 // Select the default instruction.
3417 SelectCode(N: Node);
3418}
3419
3420bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
3421 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
3422 std::vector<SDValue> &OutOps) {
3423 // Always produce a register and immediate operand, as expected by
3424 // RISCVAsmPrinter::PrintAsmMemoryOperand.
3425 switch (ConstraintID) {
3426 case InlineAsm::ConstraintCode::o:
3427 case InlineAsm::ConstraintCode::m: {
3428 SDValue Op0, Op1;
3429 [[maybe_unused]] bool Found = SelectAddrRegImm(Addr: Op, Base&: Op0, Offset&: Op1);
3430 assert(Found && "SelectAddrRegImm should always succeed");
3431 OutOps.push_back(x: Op0);
3432 OutOps.push_back(x: Op1);
3433 return false;
3434 }
3435 case InlineAsm::ConstraintCode::A:
3436 OutOps.push_back(x: Op);
3437 OutOps.push_back(
3438 x: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget->getXLenVT()));
3439 return false;
3440 default:
3441 report_fatal_error(reason: "Unexpected asm memory constraint " +
3442 InlineAsm::getMemConstraintName(C: ConstraintID));
3443 }
3444
3445 return true;
3446}
3447
3448bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
3449 SDValue &Offset) {
3450 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Addr)) {
3451 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT: Subtarget->getXLenVT());
3452 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: Subtarget->getXLenVT());
3453 return true;
3454 }
3455
3456 return false;
3457}
3458
3459// Fold constant addresses.
3460static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
3461 const MVT VT, const RISCVSubtarget *Subtarget,
3462 SDValue Addr, SDValue &Base, SDValue &Offset,
3463 bool IsPrefetch = false) {
3464 if (!isa<ConstantSDNode>(Val: Addr))
3465 return false;
3466
3467 int64_t CVal = cast<ConstantSDNode>(Val&: Addr)->getSExtValue();
3468
3469 // If the constant is a simm12, we can fold the whole constant and use X0 as
3470 // the base. If the constant can be materialized with LUI+simm12, use LUI as
3471 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
3472 int64_t Lo12 = SignExtend64<12>(x: CVal);
3473 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
3474 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
3475 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3476 return false;
3477 if (Hi) {
3478 int64_t Hi20 = (Hi >> 12) & 0xfffff;
3479 Base = SDValue(
3480 CurDAG->getMachineNode(Opcode: RISCV::LUI, dl: DL, VT,
3481 Op1: CurDAG->getTargetConstant(Val: Hi20, DL, VT)),
3482 0);
3483 } else {
3484 Base = CurDAG->getRegister(Reg: RISCV::X0, VT);
3485 }
3486 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3487 return true;
3488 }
3489
3490 // Ask how constant materialization would handle this constant.
3491 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
3492
3493 // If the last instruction would be an ADDI, we can fold its immediate and
3494 // emit the rest of the sequence as the base.
3495 if (Seq.back().getOpcode() != RISCV::ADDI)
3496 return false;
3497 Lo12 = Seq.back().getImm();
3498 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3499 return false;
3500
3501 // Drop the last instruction.
3502 Seq.pop_back();
3503 assert(!Seq.empty() && "Expected more instructions in sequence");
3504
3505 Base = selectImmSeq(CurDAG, DL, VT, Seq);
3506 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3507 return true;
3508}
3509
3510// Is this ADD instruction only used as the base pointer of scalar loads and
3511// stores?
3512static bool isWorthFoldingAdd(SDValue Add) {
3513 for (auto *User : Add->users()) {
3514 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
3515 User->getOpcode() != RISCVISD::LD_RV32 &&
3516 User->getOpcode() != RISCVISD::SD_RV32 &&
3517 User->getOpcode() != ISD::ATOMIC_LOAD &&
3518 User->getOpcode() != ISD::ATOMIC_STORE)
3519 return false;
3520 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3521 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
3522 VT != MVT::f64)
3523 return false;
3524 // Don't allow stores of the value. It must be used as the address.
3525 if (User->getOpcode() == ISD::STORE &&
3526 cast<StoreSDNode>(Val: User)->getValue() == Add)
3527 return false;
3528 if (User->getOpcode() == ISD::ATOMIC_STORE &&
3529 cast<AtomicSDNode>(Val: User)->getVal() == Add)
3530 return false;
3531 if (User->getOpcode() == RISCVISD::SD_RV32 &&
3532 (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add))
3533 return false;
3534 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
3535 return false;
3536 }
3537
3538 return true;
3539}
3540
3541bool isRegImmLoadOrStore(SDNode *User, SDValue Add) {
3542 switch (User->getOpcode()) {
3543 default:
3544 return false;
3545 case ISD::LOAD:
3546 case RISCVISD::LD_RV32:
3547 case ISD::ATOMIC_LOAD:
3548 break;
3549 case ISD::STORE:
3550 // Don't allow stores of Add. It must only be used as the address.
3551 if (cast<StoreSDNode>(Val: User)->getValue() == Add)
3552 return false;
3553 break;
3554 case RISCVISD::SD_RV32:
3555 // Don't allow stores of Add. It must only be used as the address.
3556 if (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add)
3557 return false;
3558 break;
3559 case ISD::ATOMIC_STORE:
3560 // Don't allow stores of Add. It must only be used as the address.
3561 if (cast<AtomicSDNode>(Val: User)->getVal() == Add)
3562 return false;
3563 break;
3564 }
3565
3566 return true;
3567}
3568
3569// To prevent SelectAddrRegImm from folding offsets that conflict with the
3570// fusion of PseudoMovAddr, check if the offset of every use of a given address
3571// is within the alignment.
3572bool RISCVDAGToDAGISel::areOffsetsWithinAlignment(SDValue Addr,
3573 Align Alignment) {
3574 assert(Addr->getOpcode() == RISCVISD::ADD_LO);
3575 for (auto *User : Addr->users()) {
3576 // If the user is a load or store, then the offset is 0 which is always
3577 // within alignment.
3578 if (isRegImmLoadOrStore(User, Add: Addr))
3579 continue;
3580
3581 if (CurDAG->isBaseWithConstantOffset(Op: SDValue(User, 0))) {
3582 int64_t CVal = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
3583 if (!isInt<12>(x: CVal) || Alignment <= CVal)
3584 return false;
3585
3586 // Make sure all uses are foldable load/stores.
3587 for (auto *AddUser : User->users())
3588 if (!isRegImmLoadOrStore(User: AddUser, Add: SDValue(User, 0)))
3589 return false;
3590
3591 continue;
3592 }
3593
3594 return false;
3595 }
3596
3597 return true;
3598}
3599
3600bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
3601 SDValue &Offset) {
3602 if (SelectAddrFrameIndex(Addr, Base, Offset))
3603 return true;
3604
3605 SDLoc DL(Addr);
3606 MVT VT = Addr.getSimpleValueType();
3607
3608 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
3609 bool CanFold = true;
3610 // Unconditionally fold if operand 1 is not a global address (e.g.
3611 // externsymbol)
3612 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val: Addr.getOperand(i: 1))) {
3613 const DataLayout &DL = CurDAG->getDataLayout();
3614 Align Alignment = commonAlignment(
3615 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3616 if (!areOffsetsWithinAlignment(Addr, Alignment))
3617 CanFold = false;
3618 }
3619 if (CanFold) {
3620 Base = Addr.getOperand(i: 0);
3621 Offset = Addr.getOperand(i: 1);
3622 return true;
3623 }
3624 }
3625
3626 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3627 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3628 if (isInt<12>(x: CVal)) {
3629 Base = Addr.getOperand(i: 0);
3630 if (Base.getOpcode() == RISCVISD::ADD_LO) {
3631 SDValue LoOperand = Base.getOperand(i: 1);
3632 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: LoOperand)) {
3633 // If the Lo in (ADD_LO hi, lo) is a global variable's address
3634 // (its low part, really), then we can rely on the alignment of that
3635 // variable to provide a margin of safety before low part can overflow
3636 // the 12 bits of the load/store offset. Check if CVal falls within
3637 // that margin; if so (low part + CVal) can't overflow.
3638 const DataLayout &DL = CurDAG->getDataLayout();
3639 Align Alignment = commonAlignment(
3640 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3641 if ((CVal == 0 || Alignment > CVal) &&
3642 areOffsetsWithinAlignment(Addr: Base, Alignment)) {
3643 int64_t CombinedOffset = CVal + GA->getOffset();
3644 Base = Base.getOperand(i: 0);
3645 Offset = CurDAG->getTargetGlobalAddress(
3646 GV: GA->getGlobal(), DL: SDLoc(LoOperand), VT: LoOperand.getValueType(),
3647 offset: CombinedOffset, TargetFlags: GA->getTargetFlags());
3648 return true;
3649 }
3650 }
3651 }
3652
3653 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3654 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3655 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3656 return true;
3657 }
3658 }
3659
3660 // Handle ADD with large immediates.
3661 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3662 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3663 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3664
3665 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
3666 // an ADDI for part of the offset and fold the rest into the load/store.
3667 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
3668 if (CVal >= -4096 && CVal <= 4094) {
3669 int64_t Adj = CVal < 0 ? -2048 : 2047;
3670 Base = SDValue(
3671 CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3672 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3673 0);
3674 Offset = CurDAG->getSignedTargetConstant(Val: CVal - Adj, DL, VT);
3675 return true;
3676 }
3677
3678 // For larger immediates, we might be able to save one instruction from
3679 // constant materialization by folding the Lo12 bits of the immediate into
3680 // the address. We should only do this if the ADD is only used by loads and
3681 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
3682 // separately with the full materialized immediate creating extra
3683 // instructions.
3684 if (isWorthFoldingAdd(Add: Addr) &&
3685 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3686 Offset, /*IsPrefetch=*/false)) {
3687 // Insert an ADD instruction with the materialized Hi52 bits.
3688 Base = SDValue(
3689 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3690 0);
3691 return true;
3692 }
3693 }
3694
3695 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3696 /*IsPrefetch=*/false))
3697 return true;
3698
3699 Base = Addr;
3700 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3701 return true;
3702}
3703
3704/// Similar to SelectAddrRegImm, except that the offset is a 26-bit signed
3705/// immediate. This is used by the Qualcomm Xqcilo large offset load/store
3706/// instructions (qc.e.lw/qc.e.sw), whose offset field is 26 bits wide.
3707/// Only matches offsets that do not fit a 12-bit signed immediate, so that
3708/// offsets in the simm12 range keep using the shorter (and possibly
3709/// compressible) standard load/store instructions.
3710bool RISCVDAGToDAGISel::SelectAddrRegImm26(SDValue Addr, SDValue &Base,
3711 SDValue &Offset) {
3712 SDLoc DL(Addr);
3713 MVT VT = Addr.getSimpleValueType();
3714
3715 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3716 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3717 // Fold a 26-bit (but not 12-bit) signed offset directly into the
3718 // load/store.
3719 if (isInt<26>(x: CVal) && !isInt<12>(x: CVal)) {
3720 Base = Addr.getOperand(i: 0);
3721 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3722 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3723 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3724 return true;
3725 }
3726 }
3727
3728 // The offset is just outside the 26-bit range. Split off a small (simm12)
3729 // adjustment with a plain ADDI and fold the remaining 26-bit offset into the
3730 // load/store. A plain ADDI is used (rather than the wide
3731 // qc.e.addi/qc.e.addai) because the adjustment fits simm12: this keeps it a
3732 // short, compressible (c.addi) instruction and is available without Xqcilia.
3733 //
3734 // Skip the split if the address is used other than as a foldable load/store
3735 // base. `isWorthFoldingAdd()` returns true when every user of the add node is
3736 // a scalar load/store using it as an address operand. If it return false, it
3737 // means that some use consumes the add result as a value (e.g. it feeds
3738 // another add, is a stored value, is used in arithmetic) and that use forces
3739 // the add to be materialized into a register.
3740 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)) &&
3741 isWorthFoldingAdd(Add: Addr)) {
3742 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3743 if (!isInt<26>(x: CVal)) {
3744 // check if lw in lui + add + lw combination can be compressed.
3745 // The check here purely based on the immediate value and hopes that
3746 // register allocator would assign a register from a GPRC set so that the
3747 // instruction can get compressed.
3748 bool IsLwCompressable = isShiftedUInt<5, 2>(x: CVal & ((1 << 12) - 1));
3749
3750 int64_t Imm26 = CVal < 0 ? minIntN(N: 26) : maxIntN(N: 26);
3751 int64_t Adj = CVal - Imm26;
3752 // If Adj fits within 6-bits, then both combinations will take 8 bytes
3753 // however c.addi + qc.e.lw/sw will take 1 less cycle. Also, if lw is not
3754 // compressable then both combination would take 10 bytes but again
3755 // addi + qc.e.lw/sw will take 1 less cycle.
3756 if (isInt<6>(x: Adj) || (isInt<12>(x: Adj) && !IsLwCompressable)) {
3757 Base = SDValue(CurDAG->getMachineNode(
3758 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3759 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3760 0);
3761 Offset = CurDAG->getSignedTargetConstant(Val: Imm26, DL, VT);
3762 return true;
3763 }
3764 }
3765 }
3766
3767 // Don't match: let the standard addressing modes handle it.
3768 return false;
3769}
3770
3771/// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
3772bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
3773 SDValue &Offset) {
3774 if (SelectAddrFrameIndex(Addr, Base, Offset))
3775 return true;
3776
3777 SDLoc DL(Addr);
3778 MVT VT = Addr.getSimpleValueType();
3779
3780 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3781 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3782 if (isUInt<9>(x: CVal)) {
3783 Base = Addr.getOperand(i: 0);
3784
3785 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3786 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3787 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3788 return true;
3789 }
3790 }
3791
3792 Base = Addr;
3793 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3794 return true;
3795}
3796
3797/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
3798/// Offset should be all zeros.
3799bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
3800 SDValue &Offset) {
3801 if (SelectAddrFrameIndex(Addr, Base, Offset))
3802 return true;
3803
3804 SDLoc DL(Addr);
3805 MVT VT = Addr.getSimpleValueType();
3806
3807 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3808 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3809 if (isInt<12>(x: CVal)) {
3810 Base = Addr.getOperand(i: 0);
3811
3812 // Early-out if not a valid offset.
3813 if ((CVal & 0b11111) != 0) {
3814 Base = Addr;
3815 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3816 return true;
3817 }
3818
3819 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3820 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3821 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3822 return true;
3823 }
3824 }
3825
3826 // Handle ADD with large immediates.
3827 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3828 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3829 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3830
3831 // Handle immediates in the range [-4096,-2049] or [2017, 4063]. We can save
3832 // one instruction by folding adjustment (-2048 or 2016) into the address.
3833 // The upper bound keeps CVal - 2016 within simm12 ([−2048, 2047]).
3834 if ((-2049 >= CVal && CVal >= -4096) || (4063 >= CVal && CVal >= 2017)) {
3835 int64_t Adj = CVal < 0 ? -2048 : 2016;
3836 int64_t AdjustedOffset = CVal - Adj;
3837 Base =
3838 SDValue(CurDAG->getMachineNode(
3839 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3840 Op2: CurDAG->getSignedTargetConstant(Val: AdjustedOffset, DL, VT)),
3841 0);
3842 Offset = CurDAG->getSignedTargetConstant(Val: Adj, DL, VT);
3843 return true;
3844 }
3845
3846 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3847 Offset, /*IsPrefetch=*/true)) {
3848 // Insert an ADD instruction with the materialized Hi52 bits.
3849 Base = SDValue(
3850 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3851 0);
3852 return true;
3853 }
3854 }
3855
3856 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3857 /*IsPrefetch=*/true))
3858 return true;
3859
3860 Base = Addr;
3861 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3862 return true;
3863}
3864
3865/// Return true if this a load/store that we have a RegRegScale instruction for.
3866static bool isRegRegScaleLoadOrStore(SDNode *User, SDValue Add,
3867 const RISCVSubtarget &Subtarget) {
3868 unsigned UserOpc = User->getOpcode();
3869 if (UserOpc != ISD::LOAD && UserOpc != ISD::STORE)
3870 return false;
3871 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3872 // Zilx only provides indexed loads, so it must not enable reg+reg-scale
3873 // address folding for stores. XTheadMemIdx and Xqcisls have scaled stores.
3874 bool HasScalarIntegerMemIdx =
3875 Subtarget.hasVendorXTHeadMemIdx() || Subtarget.hasVendorXqcisls() ||
3876 (Subtarget.hasStdExtZilx() && UserOpc == ISD::LOAD);
3877 if (!(VT.isScalarInteger() && HasScalarIntegerMemIdx) &&
3878 !((VT == MVT::f32 || VT == MVT::f64) &&
3879 Subtarget.hasVendorXTHeadFMemIdx()))
3880 return false;
3881 // Don't allow stores of the value. It must be used as the address.
3882 if (UserOpc == ISD::STORE && cast<StoreSDNode>(Val: User)->getValue() == Add)
3883 return false;
3884
3885 return true;
3886}
3887
3888/// Is it profitable to fold this Add into RegRegScale load/store. If \p
3889/// Shift is non-null, then we have matched a shl+add. We allow reassociating
3890/// (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2)) if there is a
3891/// single addi and we don't have a SHXADD instruction we could use.
3892/// FIXME: May still need to check how many and what kind of users the SHL has.
3893static bool isWorthFoldingIntoRegRegScale(const RISCVSubtarget &Subtarget,
3894 SDValue Add,
3895 SDValue Shift = SDValue()) {
3896 bool FoundADDI = false;
3897 for (auto *User : Add->users()) {
3898 if (isRegRegScaleLoadOrStore(User, Add, Subtarget))
3899 continue;
3900
3901 // Allow a single ADDI that is used by loads/stores if we matched a shift.
3902 if (!Shift || FoundADDI || User->getOpcode() != ISD::ADD ||
3903 !isa<ConstantSDNode>(Val: User->getOperand(Num: 1)) ||
3904 !isInt<12>(x: cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue()))
3905 return false;
3906
3907 FoundADDI = true;
3908
3909 // If we have a SHXADD instruction, prefer that over reassociating an ADDI.
3910 assert(Shift.getOpcode() == ISD::SHL);
3911 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
3912 if (Subtarget.hasShlAdd(ShAmt: ShiftAmt))
3913 return false;
3914
3915 // All users of the ADDI should be load/store.
3916 for (auto *ADDIUser : User->users())
3917 if (!isRegRegScaleLoadOrStore(User: ADDIUser, Add: SDValue(User, 0), Subtarget))
3918 return false;
3919 }
3920
3921 return true;
3922}
3923
3924bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
3925 ArrayRef<unsigned> Amounts,
3926 SDValue &Base, SDValue &Index,
3927 SDValue &Scale) {
3928 if (Addr.getOpcode() != ISD::ADD)
3929 return false;
3930 SDValue LHS = Addr.getOperand(i: 0);
3931 SDValue RHS = Addr.getOperand(i: 1);
3932
3933 EVT VT = Addr.getSimpleValueType();
3934 auto SelectShl = [this, VT, Amounts](SDValue N, SDValue &Index,
3935 SDValue &Shift) {
3936 if (N.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
3937 return false;
3938
3939 // Only match shifts by a value in range [0, MaxShiftAmount].
3940 unsigned ShiftAmt = N.getConstantOperandVal(i: 1);
3941 if (!llvm::is_contained(Range: Amounts, Element: ShiftAmt))
3942 return false;
3943
3944 Index = N.getOperand(i: 0);
3945 Shift = CurDAG->getTargetConstant(Val: ShiftAmt, DL: SDLoc(N), VT);
3946 return true;
3947 };
3948
3949 if (auto *C1 = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3950 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
3951 if (LHS.getOpcode() == ISD::ADD &&
3952 !isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
3953 isInt<12>(x: C1->getSExtValue())) {
3954 if (SelectShl(LHS.getOperand(i: 1), Index, Scale) &&
3955 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 1))) {
3956 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
3957 DL: SDLoc(Addr), VT);
3958 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
3959 Op1: LHS.getOperand(i: 0), Op2: C1Val),
3960 0);
3961 return true;
3962 }
3963
3964 // Add is commutative so we need to check both operands.
3965 if (SelectShl(LHS.getOperand(i: 0), Index, Scale) &&
3966 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 0))) {
3967 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
3968 DL: SDLoc(Addr), VT);
3969 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
3970 Op1: LHS.getOperand(i: 1), Op2: C1Val),
3971 0);
3972 return true;
3973 }
3974 }
3975
3976 // Don't match add with constants.
3977 // FIXME: Is this profitable for large constants that have 0s in the lower
3978 // 12 bits that we can materialize with LUI?
3979 return false;
3980 }
3981
3982 // Try to match a shift on the RHS.
3983 if (SelectShl(RHS, Index, Scale)) {
3984 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: RHS))
3985 return false;
3986 Base = LHS;
3987 return true;
3988 }
3989
3990 // Try to match a shift on the LHS.
3991 if (SelectShl(LHS, Index, Scale)) {
3992 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: LHS))
3993 return false;
3994 Base = RHS;
3995 return true;
3996 }
3997
3998 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr))
3999 return false;
4000
4001 // Bail out if 0 is not in candidate shift amounts.
4002 if (!llvm::is_contained(Range&: Amounts, Element: 0))
4003 return false;
4004
4005 Base = LHS;
4006 Index = RHS;
4007 Scale = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT);
4008 return true;
4009}
4010
4011bool RISCVDAGToDAGISel::SelectAddrRegZextRegScale(SDValue Addr,
4012 ArrayRef<unsigned> Amounts,
4013 unsigned Bits, SDValue &Base,
4014 SDValue &Index,
4015 SDValue &Scale) {
4016 if (!SelectAddrRegRegScale(Addr, Amounts, Base, Index, Scale))
4017 return false;
4018
4019 if (Index.getOpcode() == ISD::AND) {
4020 auto *C = dyn_cast<ConstantSDNode>(Val: Index.getOperand(i: 1));
4021 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4022 Index = Index.getOperand(i: 0);
4023 return true;
4024 }
4025 }
4026
4027 return false;
4028}
4029
4030bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base,
4031 SDValue &Offset) {
4032 if (Addr.getOpcode() != ISD::ADD)
4033 return false;
4034
4035 if (isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)))
4036 return false;
4037
4038 Base = Addr.getOperand(i: 0);
4039 Offset = Addr.getOperand(i: 1);
4040 return true;
4041}
4042
4043bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
4044 SDValue &ShAmt) {
4045 ShAmt = N;
4046
4047 // Peek through zext.
4048 if (ShAmt->getOpcode() == ISD::ZERO_EXTEND)
4049 ShAmt = ShAmt.getOperand(i: 0);
4050
4051 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
4052 // amount. If there is an AND on the shift amount, we can bypass it if it
4053 // doesn't affect any of those bits.
4054 if (ShAmt.getOpcode() == ISD::AND &&
4055 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4056 const APInt &AndMask = ShAmt.getConstantOperandAPInt(i: 1);
4057
4058 // Since the max shift amount is a power of 2 we can subtract 1 to make a
4059 // mask that covers the bits needed to represent all shift amounts.
4060 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
4061 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
4062
4063 if (ShMask.isSubsetOf(RHS: AndMask)) {
4064 ShAmt = ShAmt.getOperand(i: 0);
4065 } else {
4066 // SimplifyDemandedBits may have optimized the mask so try restoring any
4067 // bits that are known zero.
4068 KnownBits Known = CurDAG->computeKnownBits(Op: ShAmt.getOperand(i: 0));
4069 if (!ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
4070 return true;
4071 ShAmt = ShAmt.getOperand(i: 0);
4072 }
4073 }
4074
4075 if (ShAmt.getOpcode() == ISD::ADD &&
4076 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4077 uint64_t Imm = ShAmt.getConstantOperandVal(i: 1);
4078 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
4079 // to avoid the ADD.
4080 if (Imm != 0 && Imm % ShiftWidth == 0) {
4081 ShAmt = ShAmt.getOperand(i: 0);
4082 return true;
4083 }
4084 } else if (ShAmt.getOpcode() == ISD::SUB &&
4085 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 0))) {
4086 uint64_t Imm = ShAmt.getConstantOperandVal(i: 0);
4087 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
4088 // generate a NEG instead of a SUB of a constant.
4089 if (Imm != 0 && Imm % ShiftWidth == 0) {
4090 SDLoc DL(ShAmt);
4091 EVT VT = ShAmt.getValueType();
4092 SDValue Zero = CurDAG->getRegister(Reg: RISCV::X0, VT);
4093 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
4094 MachineSDNode *Neg = CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero,
4095 Op2: ShAmt.getOperand(i: 1));
4096 ShAmt = SDValue(Neg, 0);
4097 return true;
4098 }
4099 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4100 // to generate a NOT instead of a SUB of a constant.
4101 if (Imm % ShiftWidth == ShiftWidth - 1) {
4102 SDLoc DL(ShAmt);
4103 EVT VT = ShAmt.getValueType();
4104 MachineSDNode *Not = CurDAG->getMachineNode(
4105 Opcode: RISCV::XORI, dl: DL, VT, Op1: ShAmt.getOperand(i: 1),
4106 Op2: CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/IsTarget: true));
4107 ShAmt = SDValue(Not, 0);
4108 return true;
4109 }
4110 }
4111
4112 return true;
4113}
4114
4115/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
4116/// check for equality with 0. This function emits instructions that convert the
4117/// seteq/setne into something that can be compared with 0.
4118/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
4119/// ISD::SETNE).
4120bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
4121 SDValue &Val, bool OneUse) {
4122 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
4123 "Unexpected condition code!");
4124
4125 // We're looking for a setcc.
4126 if (N->getOpcode() != ISD::SETCC)
4127 return false;
4128
4129 if (OneUse && !N->hasOneUse())
4130 return false;
4131
4132 // Must be an equality comparison.
4133 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
4134 if (CCVal != ExpectedCCVal)
4135 return false;
4136
4137 SDValue LHS = N->getOperand(Num: 0);
4138 SDValue RHS = N->getOperand(Num: 1);
4139
4140 if (!LHS.getValueType().isScalarInteger())
4141 return false;
4142
4143 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
4144 if (isNullConstant(V: RHS)) {
4145 Val = LHS;
4146 return true;
4147 }
4148
4149 SDLoc DL(N);
4150
4151 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4152 int64_t CVal = C->getSExtValue();
4153 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
4154 // non-zero otherwise.
4155 if (CVal == -2048) {
4156 Val = SDValue(
4157 CurDAG->getMachineNode(
4158 Opcode: RISCV::XORI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4159 Op2: CurDAG->getSignedTargetConstant(Val: CVal, DL, VT: N->getValueType(ResNo: 0))),
4160 0);
4161 return true;
4162 }
4163 // If the RHS is [-2047,2048], we can use addi/addiw with -RHS to produce 0
4164 // if the LHS is equal to the RHS and non-zero otherwise.
4165 if (isInt<12>(x: CVal) || CVal == 2048) {
4166 unsigned Opc = RISCV::ADDI;
4167 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4168 cast<VTSDNode>(Val: LHS.getOperand(i: 1))->getVT() == MVT::i32) {
4169 Opc = RISCV::ADDIW;
4170 LHS = LHS.getOperand(i: 0);
4171 }
4172
4173 Val = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4174 Op2: CurDAG->getSignedTargetConstant(
4175 Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4176 0);
4177 return true;
4178 }
4179 if (isPowerOf2_64(Value: CVal) && Subtarget->hasStdExtZbs()) {
4180 Val = SDValue(
4181 CurDAG->getMachineNode(
4182 Opcode: RISCV::BINVI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4183 Op2: CurDAG->getTargetConstant(Val: Log2_64(Value: CVal), DL, VT: N->getValueType(ResNo: 0))),
4184 0);
4185 return true;
4186 }
4187 // Same as the addi case above but for larger immediates (signed 26-bit) use
4188 // the QC_E_ADDI instruction from the Xqcilia extension, if available. Avoid
4189 // anything which can be done with a single lui as it might be compressible.
4190 if (Subtarget->hasVendorXqcilia() && isInt<26>(x: CVal) &&
4191 (CVal & 0xFFF) != 0) {
4192 Val = SDValue(
4193 CurDAG->getMachineNode(
4194 Opcode: RISCV::QC_E_ADDI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4195 Op2: CurDAG->getSignedTargetConstant(Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4196 0);
4197 return true;
4198 }
4199 }
4200
4201 // If nothing else we can XOR the LHS and RHS to produce zero if they are
4202 // equal and a non-zero value if they aren't.
4203 Val = SDValue(
4204 CurDAG->getMachineNode(Opcode: RISCV::XOR, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS, Op2: RHS), 0);
4205 return true;
4206}
4207
4208bool RISCVDAGToDAGISel::selectSExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4209 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4210 cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT().getSizeInBits() == Bits) {
4211 Val = N.getOperand(i: 0);
4212 return true;
4213 }
4214
4215 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
4216 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
4217 return N;
4218
4219 SDValue N0 = N.getOperand(i: 0);
4220 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4221 N.getConstantOperandVal(i: 1) == ShiftAmt &&
4222 N0.getConstantOperandVal(i: 1) == ShiftAmt)
4223 return N0.getOperand(i: 0);
4224
4225 return N;
4226 };
4227
4228 MVT VT = N.getSimpleValueType();
4229 if (CurDAG->ComputeNumSignBits(Op: N) > (VT.getSizeInBits() - Bits)) {
4230 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
4231 return true;
4232 }
4233
4234 return false;
4235}
4236
4237bool RISCVDAGToDAGISel::selectZExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4238 if (N.getOpcode() == ISD::AND) {
4239 auto *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
4240 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4241 Val = N.getOperand(i: 0);
4242 return true;
4243 }
4244 }
4245 MVT VT = N.getSimpleValueType();
4246 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: Bits);
4247 if (CurDAG->MaskedValueIsZero(Op: N, Mask)) {
4248 Val = N;
4249 return true;
4250 }
4251
4252 return false;
4253}
4254
4255/// Look for various patterns that can be done with a SHL that can be folded
4256/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
4257/// SHXADD we are trying to match.
4258bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
4259 SDValue &Val) {
4260 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4261 SDValue N0 = N.getOperand(i: 0);
4262
4263 if (bool LeftShift = N0.getOpcode() == ISD::SHL;
4264 (LeftShift || N0.getOpcode() == ISD::SRL) &&
4265 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4266 uint64_t Mask = N.getConstantOperandVal(i: 1);
4267 unsigned C2 = N0.getConstantOperandVal(i: 1);
4268
4269 unsigned XLen = Subtarget->getXLen();
4270 if (LeftShift)
4271 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4272 else
4273 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
4274
4275 if (isShiftedMask_64(Value: Mask)) {
4276 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4277 unsigned Trailing = llvm::countr_zero(Val: Mask);
4278 if (Trailing != ShAmt)
4279 return false;
4280
4281 unsigned Opcode;
4282 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
4283 // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
4284 // followed by a SHXADD with c3 for the X amount.
4285 if (LeftShift && Leading == 0 && C2 < Trailing)
4286 Opcode = RISCV::SRLI;
4287 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
4288 // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
4289 // followed by a SHXADD with c3 for the X amount.
4290 else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
4291 Opcode = RISCV::SRLIW;
4292 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
4293 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
4294 // followed by a SHXADD using c3 for the X amount.
4295 else if (!LeftShift && Leading == C2)
4296 Opcode = RISCV::SRLI;
4297 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
4298 // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
4299 // followed by a SHXADD using c3 for the X amount.
4300 else if (!LeftShift && Leading == 32 + C2)
4301 Opcode = RISCV::SRLIW;
4302 else
4303 return false;
4304
4305 SDLoc DL(N);
4306 EVT VT = N.getValueType();
4307 ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
4308 Val = SDValue(
4309 CurDAG->getMachineNode(Opcode, dl: DL, VT, Op1: N0.getOperand(i: 0),
4310 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT)),
4311 0);
4312 return true;
4313 }
4314 } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
4315 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4316 uint64_t Mask = N.getConstantOperandVal(i: 1);
4317 unsigned C2 = N0.getConstantOperandVal(i: 1);
4318
4319 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
4320 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
4321 // use (srli (srai y, c2 - c3), c3 + c4) followed by a SHXADD with c4 as
4322 // the X amount.
4323 if (isShiftedMask_64(Value: Mask)) {
4324 unsigned XLen = Subtarget->getXLen();
4325 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4326 unsigned Trailing = llvm::countr_zero(Val: Mask);
4327 if (C2 > Leading && Leading > 0 && Trailing == ShAmt) {
4328 SDLoc DL(N);
4329 EVT VT = N.getValueType();
4330 Val = SDValue(CurDAG->getMachineNode(
4331 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4332 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT)),
4333 0);
4334 Val = SDValue(CurDAG->getMachineNode(
4335 Opcode: RISCV::SRLI, dl: DL, VT, Op1: Val,
4336 Op2: CurDAG->getTargetConstant(Val: Leading + ShAmt, DL, VT)),
4337 0);
4338 return true;
4339 }
4340 }
4341 }
4342 } else if (bool LeftShift = N.getOpcode() == ISD::SHL;
4343 (LeftShift || N.getOpcode() == ISD::SRL) &&
4344 isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4345 SDValue N0 = N.getOperand(i: 0);
4346 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
4347 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4348 uint64_t Mask = N0.getConstantOperandVal(i: 1);
4349 if (isShiftedMask_64(Value: Mask)) {
4350 unsigned C1 = N.getConstantOperandVal(i: 1);
4351 unsigned XLen = Subtarget->getXLen();
4352 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4353 unsigned Trailing = llvm::countr_zero(Val: Mask);
4354 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
4355 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
4356 if (LeftShift && Leading == 32 && Trailing > 0 &&
4357 (Trailing + C1) == ShAmt) {
4358 SDLoc DL(N);
4359 EVT VT = N.getValueType();
4360 Val = SDValue(CurDAG->getMachineNode(
4361 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4362 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4363 0);
4364 return true;
4365 }
4366 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
4367 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
4368 if (!LeftShift && Leading == 32 && Trailing > C1 &&
4369 (Trailing - C1) == ShAmt) {
4370 SDLoc DL(N);
4371 EVT VT = N.getValueType();
4372 Val = SDValue(CurDAG->getMachineNode(
4373 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4374 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4375 0);
4376 return true;
4377 }
4378 }
4379 }
4380 }
4381
4382 return false;
4383}
4384
4385/// Look for various patterns that can be done with a SHL that can be folded
4386/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
4387/// SHXADD_UW we are trying to match.
4388bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
4389 SDValue &Val) {
4390 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
4391 N.hasOneUse()) {
4392 SDValue N0 = N.getOperand(i: 0);
4393 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4394 N0.hasOneUse()) {
4395 uint64_t Mask = N.getConstantOperandVal(i: 1);
4396 unsigned C2 = N0.getConstantOperandVal(i: 1);
4397
4398 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4399
4400 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
4401 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
4402 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
4403 if (isShiftedMask_64(Value: Mask)) {
4404 unsigned Leading = llvm::countl_zero(Val: Mask);
4405 unsigned Trailing = llvm::countr_zero(Val: Mask);
4406 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
4407 SDLoc DL(N);
4408 EVT VT = N.getValueType();
4409 Val = SDValue(CurDAG->getMachineNode(
4410 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4411 Op2: CurDAG->getTargetConstant(Val: C2 - ShAmt, DL, VT)),
4412 0);
4413 return true;
4414 }
4415 }
4416 }
4417 }
4418
4419 return false;
4420}
4421
4422bool RISCVDAGToDAGISel::orDisjoint(const SDNode *N) const {
4423 assert(N->getOpcode() == ISD::OR || N->getOpcode() == RISCVISD::OR_VL);
4424 if (N->getFlags().hasDisjoint())
4425 return true;
4426 return CurDAG->haveNoCommonBitsSet(A: N->getOperand(Num: 0), B: N->getOperand(Num: 1));
4427}
4428
4429bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
4430 SDValue N, SDValue &Val) {
4431 int OrigCost = RISCVMatInt::getIntMatCost(Val: APInt(64, OrigImm), Size: 64, STI: *Subtarget,
4432 /*CompressionCost=*/true);
4433 int Cost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
4434 /*CompressionCost=*/true);
4435 if (OrigCost <= Cost)
4436 return false;
4437
4438 Val = selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm, Subtarget: *Subtarget);
4439 return true;
4440}
4441
4442bool RISCVDAGToDAGISel::selectZExtImm32(SDValue N, SDValue &Val) {
4443 if (!isa<ConstantSDNode>(Val: N))
4444 return false;
4445 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4446 if ((Imm >> 31) != 1)
4447 return false;
4448
4449 for (const SDNode *U : N->users()) {
4450 switch (U->getOpcode()) {
4451 case ISD::ADD:
4452 break;
4453 case ISD::OR:
4454 if (orDisjoint(N: U))
4455 break;
4456 return false;
4457 default:
4458 return false;
4459 }
4460 }
4461
4462 return selectImm64IfCheaper(Imm: 0xffffffff00000000 | Imm, OrigImm: Imm, N, Val);
4463}
4464
4465bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
4466 if (!isa<ConstantSDNode>(Val: N))
4467 return false;
4468 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4469 if (isInt<32>(x: Imm))
4470 return false;
4471 if (Imm == INT64_MIN)
4472 return false;
4473
4474 for (const SDNode *U : N->users()) {
4475 switch (U->getOpcode()) {
4476 case ISD::ADD:
4477 break;
4478 case RISCVISD::VMV_V_X_VL:
4479 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4480 return V->getOpcode() == ISD::ADD ||
4481 V->getOpcode() == RISCVISD::ADD_VL;
4482 }))
4483 return false;
4484 break;
4485 default:
4486 return false;
4487 }
4488 }
4489
4490 return selectImm64IfCheaper(Imm: -Imm, OrigImm: Imm, N, Val);
4491}
4492
4493bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
4494 if (!isa<ConstantSDNode>(Val: N))
4495 return false;
4496 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4497
4498 // For 32-bit signed constants, we can only substitute LUI+ADDI with LUI.
4499 if (isInt<32>(x: Imm) && ((Imm & 0xfff) != 0xfff || Imm == -1))
4500 return false;
4501
4502 // Abandon this transform if the constant is needed elsewhere.
4503 for (const SDNode *U : N->users()) {
4504 switch (U->getOpcode()) {
4505 case ISD::AND:
4506 case ISD::OR:
4507 case ISD::XOR:
4508 if (!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()))
4509 return false;
4510 break;
4511 case RISCVISD::VMV_V_X_VL:
4512 if (!Subtarget->hasStdExtZvkb())
4513 return false;
4514 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4515 return V->getOpcode() == ISD::AND ||
4516 V->getOpcode() == RISCVISD::AND_VL;
4517 }))
4518 return false;
4519 break;
4520 default:
4521 return false;
4522 }
4523 }
4524
4525 if (isInt<32>(x: Imm)) {
4526 Val =
4527 selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm: ~Imm, Subtarget: *Subtarget);
4528 return true;
4529 }
4530
4531 // For 64-bit constants, the instruction sequences get complex,
4532 // so we select inverted only if it's cheaper.
4533 return selectImm64IfCheaper(Imm: ~Imm, OrigImm: Imm, N, Val);
4534}
4535
4536static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
4537 unsigned Bits,
4538 const TargetInstrInfo *TII) {
4539 unsigned MCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: User->getMachineOpcode());
4540
4541 if (!MCOpcode)
4542 return false;
4543
4544 const MCInstrDesc &MCID = TII->get(Opcode: User->getMachineOpcode());
4545 const uint64_t TSFlags = MCID.TSFlags;
4546 if (!RISCVII::hasSEWOp(TSFlags))
4547 return false;
4548 assert(RISCVII::hasVLOp(TSFlags));
4549
4550 unsigned ChainOpIdx = User->getNumOperands() - 1;
4551 bool HasChainOp = User->getOperand(Num: ChainOpIdx).getValueType() == MVT::Other;
4552 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
4553 unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
4554 const unsigned Log2SEW = User->getConstantOperandVal(Num: VLIdx + 1);
4555
4556 if (UserOpNo == VLIdx)
4557 return false;
4558
4559 auto NumDemandedBits =
4560 RISCV::getVectorLowDemandedScalarBits(Opcode: MCOpcode, Log2SEW);
4561 return NumDemandedBits && Bits >= *NumDemandedBits;
4562}
4563
4564// Return true if all users of this SDNode* only consume the lower \p Bits.
4565// This can be used to form W instructions for add/sub/mul/shl even when the
4566// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
4567// SimplifyDemandedBits has made it so some users see a sext_inreg and some
4568// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
4569// the add/sub/mul/shl to become non-W instructions. By checking the users we
4570// may be able to use a W instruction and CSE with the other instruction if
4571// this has happened. We could try to detect that the CSE opportunity exists
4572// before doing this, but that would be more complicated.
4573bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits,
4574 const unsigned Depth) const {
4575 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
4576 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
4577 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
4578 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
4579 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
4580 isa<ConstantSDNode>(Node) || Depth != 0) &&
4581 "Unexpected opcode");
4582
4583 if (Depth >= SelectionDAG::MaxRecursionDepth)
4584 return false;
4585
4586 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
4587 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
4588 if (Depth == 0 && !Node->getValueType(ResNo: 0).isScalarInteger())
4589 return false;
4590
4591 for (SDUse &Use : Node->uses()) {
4592 SDNode *User = Use.getUser();
4593 // Users of this node should have already been instruction selected
4594 if (!User->isMachineOpcode())
4595 return false;
4596
4597 // TODO: Add more opcodes?
4598 switch (User->getMachineOpcode()) {
4599 default:
4600 if (vectorPseudoHasAllNBitUsers(User, UserOpNo: Use.getOperandNo(), Bits, TII))
4601 break;
4602 return false;
4603 case RISCV::ADDW:
4604 case RISCV::ADDIW:
4605 case RISCV::SUBW:
4606 case RISCV::MULW:
4607 case RISCV::SLLW:
4608 case RISCV::SLLIW:
4609 case RISCV::SRAW:
4610 case RISCV::SRAIW:
4611 case RISCV::SRLW:
4612 case RISCV::SRLIW:
4613 case RISCV::DIVW:
4614 case RISCV::DIVUW:
4615 case RISCV::REMW:
4616 case RISCV::REMUW:
4617 case RISCV::ROLW:
4618 case RISCV::RORW:
4619 case RISCV::RORIW:
4620 case RISCV::CLSW:
4621 case RISCV::CLZW:
4622 case RISCV::CTZW:
4623 case RISCV::CPOPW:
4624 case RISCV::SLLI_UW:
4625 case RISCV::ABSW:
4626 case RISCV::FMV_W_X:
4627 case RISCV::FCVT_H_W:
4628 case RISCV::FCVT_H_W_INX:
4629 case RISCV::FCVT_H_WU:
4630 case RISCV::FCVT_H_WU_INX:
4631 case RISCV::FCVT_S_W:
4632 case RISCV::FCVT_S_W_INX:
4633 case RISCV::FCVT_S_WU:
4634 case RISCV::FCVT_S_WU_INX:
4635 case RISCV::FCVT_D_W:
4636 case RISCV::FCVT_D_W_INX:
4637 case RISCV::FCVT_D_WU:
4638 case RISCV::FCVT_D_WU_INX:
4639 case RISCV::TH_REVW:
4640 case RISCV::TH_SRRIW:
4641 if (Bits >= 32)
4642 break;
4643 return false;
4644 case RISCV::SLL:
4645 case RISCV::SRA:
4646 case RISCV::SRL:
4647 case RISCV::ROL:
4648 case RISCV::ROR:
4649 case RISCV::BSET:
4650 case RISCV::BCLR:
4651 case RISCV::BINV:
4652 // Shift amount operands only use log2(Xlen) bits.
4653 if (Use.getOperandNo() == 1 && Bits >= Log2_32(Value: Subtarget->getXLen()))
4654 break;
4655 return false;
4656 case RISCV::SLLI:
4657 // SLLI only uses the lower (XLen - ShAmt) bits.
4658 if (Bits >= Subtarget->getXLen() - User->getConstantOperandVal(Num: 1))
4659 break;
4660 return false;
4661 case RISCV::ANDI:
4662 if (Bits >= (unsigned)llvm::bit_width(Value: User->getConstantOperandVal(Num: 1)))
4663 break;
4664 goto RecCheck;
4665 case RISCV::ORI: {
4666 uint64_t Imm = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
4667 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(Value: ~Imm))
4668 break;
4669 [[fallthrough]];
4670 }
4671 case RISCV::AND:
4672 case RISCV::OR:
4673 case RISCV::XOR:
4674 case RISCV::XORI:
4675 case RISCV::ANDN:
4676 case RISCV::ORN:
4677 case RISCV::XNOR:
4678 case RISCV::SH1ADD:
4679 case RISCV::SH2ADD:
4680 case RISCV::SH3ADD:
4681 RecCheck:
4682 if (hasAllNBitUsers(Node: User, Bits, Depth: Depth + 1))
4683 break;
4684 return false;
4685 case RISCV::SRLI: {
4686 unsigned ShAmt = User->getConstantOperandVal(Num: 1);
4687 // If we are shifting right by less than Bits, and users don't demand any
4688 // bits that were shifted into [Bits-1:0], then we can consider this as an
4689 // N-Bit user.
4690 if (Bits > ShAmt && hasAllNBitUsers(Node: User, Bits: Bits - ShAmt, Depth: Depth + 1))
4691 break;
4692 return false;
4693 }
4694 case RISCV::SEXT_B:
4695 case RISCV::PACKH:
4696 if (Bits >= 8)
4697 break;
4698 return false;
4699 case RISCV::SEXT_H:
4700 case RISCV::FMV_H_X:
4701 case RISCV::ZEXT_H_RV32:
4702 case RISCV::ZEXT_H_RV64:
4703 case RISCV::PACKW:
4704 if (Bits >= 16)
4705 break;
4706 return false;
4707 case RISCV::PACK:
4708 if (Bits >= (Subtarget->getXLen() / 2))
4709 break;
4710 return false;
4711 case RISCV::PPAIRE_H:
4712 // If only the lower 32-bits of the result are used, then only the
4713 // lower 16 bits of the inputs are used.
4714 if (Bits >= 16 && hasAllNBitUsers(Node: User, Bits: 32, Depth: Depth + 1))
4715 break;
4716 return false;
4717 case RISCV::ADD_UW:
4718 case RISCV::SH1ADD_UW:
4719 case RISCV::SH2ADD_UW:
4720 case RISCV::SH3ADD_UW:
4721 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
4722 // 32 bits.
4723 if (Use.getOperandNo() == 0 && Bits >= 32)
4724 break;
4725 return false;
4726 case RISCV::SB:
4727 if (Use.getOperandNo() == 0 && Bits >= 8)
4728 break;
4729 return false;
4730 case RISCV::SH:
4731 if (Use.getOperandNo() == 0 && Bits >= 16)
4732 break;
4733 return false;
4734 case RISCV::SW:
4735 if (Use.getOperandNo() == 0 && Bits >= 32)
4736 break;
4737 return false;
4738 case RISCV::TH_EXT:
4739 case RISCV::TH_EXTU: {
4740 unsigned Msb = User->getConstantOperandVal(Num: 1);
4741 unsigned Lsb = User->getConstantOperandVal(Num: 2);
4742 // Behavior of Msb < Lsb is not well documented.
4743 if (Msb >= Lsb && Bits > Msb)
4744 break;
4745 return false;
4746 }
4747 }
4748 }
4749
4750 return true;
4751}
4752
4753// Select a constant that can be represented as (sign_extend(imm5) << imm2).
4754bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
4755 SDValue &Shl2) {
4756 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4757 if (!C)
4758 return false;
4759
4760 int64_t Offset = C->getSExtValue();
4761 for (unsigned Shift = 0; Shift < 4; Shift++) {
4762 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0)) {
4763 EVT VT = N->getValueType(ResNo: 0);
4764 Simm5 = CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(N), VT);
4765 Shl2 = CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(N), VT);
4766 return true;
4767 }
4768 }
4769
4770 return false;
4771}
4772
4773// Select VL as a 5 bit immediate or a value that will become a register. This
4774// allows us to choose between VSETIVLI or VSETVLI later.
4775bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
4776 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4777 if (C && isUInt<5>(x: C->getZExtValue())) {
4778 VL = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(N),
4779 VT: N->getValueType(ResNo: 0));
4780 } else if (C && C->isAllOnes()) {
4781 // Treat all ones as VLMax.
4782 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4783 VT: N->getValueType(ResNo: 0));
4784 } else if (isa<RegisterSDNode>(Val: N) &&
4785 cast<RegisterSDNode>(Val&: N)->getReg() == RISCV::X0) {
4786 // All our VL operands use an operand that allows GPRNoX0 or an immediate
4787 // as the register class. Convert X0 to a special immediate to pass the
4788 // MachineVerifier. This is recognized specially by the vsetvli insertion
4789 // pass.
4790 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4791 VT: N->getValueType(ResNo: 0));
4792 } else {
4793 VL = N;
4794 }
4795
4796 return true;
4797}
4798
4799static SDValue findVSplat(SDValue N) {
4800 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
4801 if (!N.getOperand(i: 0).isUndef())
4802 return SDValue();
4803 N = N.getOperand(i: 1);
4804 }
4805 SDValue Splat = N;
4806 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
4807 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
4808 !Splat.getOperand(i: 0).isUndef())
4809 return SDValue();
4810 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
4811 return Splat;
4812}
4813
4814bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
4815 SDValue Splat = findVSplat(N);
4816 if (!Splat)
4817 return false;
4818
4819 SplatVal = Splat.getOperand(i: 1);
4820 return true;
4821}
4822
4823static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
4824 SelectionDAG &DAG,
4825 const RISCVSubtarget &Subtarget,
4826 std::function<bool(int64_t)> ValidateImm,
4827 bool Decrement = false) {
4828 SDValue Splat = findVSplat(N);
4829 if (!Splat || !isa<ConstantSDNode>(Val: Splat.getOperand(i: 1)))
4830 return false;
4831
4832 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
4833 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
4834 "Unexpected splat operand type");
4835
4836 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
4837 // type is wider than the resulting vector element type: an implicit
4838 // truncation first takes place. Therefore, perform a manual
4839 // truncation/sign-extension in order to ignore any truncated bits and catch
4840 // any zero-extended immediate.
4841 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
4842 // sign-extending to (XLenVT -1).
4843 APInt SplatConst = Splat.getConstantOperandAPInt(i: 1).sextOrTrunc(width: SplatEltSize);
4844
4845 int64_t SplatImm = SplatConst.getSExtValue();
4846
4847 if (!ValidateImm(SplatImm))
4848 return false;
4849
4850 if (Decrement)
4851 SplatImm -= 1;
4852
4853 SplatVal =
4854 DAG.getSignedTargetConstant(Val: SplatImm, DL: SDLoc(N), VT: Subtarget.getXLenVT());
4855 return true;
4856}
4857
4858bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
4859 return selectVSplatImmHelper(N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4860 ValidateImm: [](int64_t Imm) { return isInt<5>(x: Imm); });
4861}
4862
4863bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
4864 return selectVSplatImmHelper(
4865 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4866 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4867 /*Decrement=*/true);
4868}
4869
4870bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal) {
4871 return selectVSplatImmHelper(
4872 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4873 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4874 /*Decrement=*/false);
4875}
4876
4877bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
4878 SDValue &SplatVal) {
4879 return selectVSplatImmHelper(
4880 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4881 ValidateImm: [](int64_t Imm) { return Imm != 0 && Imm >= -15 && Imm <= 16; },
4882 /*Decrement=*/true);
4883}
4884
4885bool RISCVDAGToDAGISel::selectVSplatUimm(SDValue N, unsigned Bits,
4886 SDValue &SplatVal) {
4887 return selectVSplatImmHelper(
4888 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4889 ValidateImm: [Bits](int64_t Imm) { return isUIntN(N: Bits, x: Imm); });
4890}
4891
4892bool RISCVDAGToDAGISel::selectVSplatImm64Neg(SDValue N, SDValue &SplatVal) {
4893 SDValue Splat = findVSplat(N);
4894 return Splat && selectNegImm(N: Splat.getOperand(i: 1), Val&: SplatVal);
4895}
4896
4897bool RISCVDAGToDAGISel::selectLow8BitsVSplat(SDValue N, SDValue &SplatVal) {
4898 auto IsExtOrTrunc = [](SDValue N) {
4899 switch (N->getOpcode()) {
4900 case ISD::SIGN_EXTEND:
4901 case ISD::ZERO_EXTEND:
4902 // There's no passthru on these _VL nodes so any VL/mask is ok, since any
4903 // inactive elements will be undef.
4904 case RISCVISD::TRUNCATE_VECTOR_VL:
4905 case RISCVISD::VSEXT_VL:
4906 case RISCVISD::VZEXT_VL:
4907 return true;
4908 default:
4909 return false;
4910 }
4911 };
4912
4913 // We can have multiple nested nodes, so unravel them all if needed.
4914 while (IsExtOrTrunc(N)) {
4915 if (!N.hasOneUse() || N.getScalarValueSizeInBits() < 8)
4916 return false;
4917 N = N->getOperand(Num: 0);
4918 }
4919
4920 return selectVSplat(N, SplatVal);
4921}
4922
4923bool RISCVDAGToDAGISel::selectScalarFPAsInt(SDValue N, SDValue &Imm) {
4924 // Allow bitcasts from XLenVT -> FP.
4925 if (N.getOpcode() == ISD::BITCAST &&
4926 N.getOperand(i: 0).getValueType() == Subtarget->getXLenVT()) {
4927 Imm = N.getOperand(i: 0);
4928 return true;
4929 }
4930 // Allow moves from XLenVT to FP.
4931 if (N.getOpcode() == RISCVISD::FMV_H_X ||
4932 N.getOpcode() == RISCVISD::FMV_W_X_RV64) {
4933 Imm = N.getOperand(i: 0);
4934 return true;
4935 }
4936
4937 // Otherwise, look for FP constants that can materialized with scalar int.
4938 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val: N.getNode());
4939 if (!CFP)
4940 return false;
4941 const APFloat &APF = CFP->getValueAPF();
4942 // td can handle +0.0 already.
4943 if (APF.isPosZero())
4944 return false;
4945
4946 MVT VT = CFP->getSimpleValueType(ResNo: 0);
4947
4948 MVT XLenVT = Subtarget->getXLenVT();
4949 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
4950 assert(APF.isNegZero() && "Unexpected constant.");
4951 return false;
4952 }
4953 SDLoc DL(N);
4954 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
4955 Subtarget: *Subtarget);
4956 return true;
4957}
4958
4959bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
4960 SDValue &Imm) {
4961 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N)) {
4962 int64_t ImmVal = SignExtend64(X: C->getSExtValue(), B: Width);
4963
4964 if (!isInt<5>(x: ImmVal))
4965 return false;
4966
4967 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N),
4968 VT: Subtarget->getXLenVT());
4969 return true;
4970 }
4971
4972 return false;
4973}
4974
4975// Match XOR with a VMSET_VL operand. Return the other operand.
4976bool RISCVDAGToDAGISel::selectVMNOTOp(SDValue N, SDValue &Res) {
4977 if (N.getOpcode() != ISD::XOR)
4978 return false;
4979
4980 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
4981 Res = N.getOperand(i: 1);
4982 return true;
4983 }
4984
4985 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
4986 Res = N.getOperand(i: 0);
4987 return true;
4988 }
4989
4990 return false;
4991}
4992
4993// Match VMXOR_VL with a VMSET_VL operand. Making sure that that VL operand
4994// matches the parent's VL. Return the other operand of the VMXOR_VL.
4995bool RISCVDAGToDAGISel::selectVMNOT_VLOp(SDNode *Parent, SDValue N,
4996 SDValue &Res) {
4997 if (N.getOpcode() != RISCVISD::VMXOR_VL)
4998 return false;
4999
5000 assert(Parent &&
5001 (Parent->getOpcode() == RISCVISD::VMAND_VL ||
5002 Parent->getOpcode() == RISCVISD::VMOR_VL ||
5003 Parent->getOpcode() == RISCVISD::VMXOR_VL) &&
5004 "Unexpected parent");
5005
5006 // The VL should match the parent.
5007 if (Parent->getOperand(Num: 2) != N->getOperand(Num: 2))
5008 return false;
5009
5010 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5011 Res = N.getOperand(i: 1);
5012 return true;
5013 }
5014
5015 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5016 Res = N.getOperand(i: 0);
5017 return true;
5018 }
5019
5020 return false;
5021}
5022
5023// Try to remove sext.w if the input is a W instruction or can be made into
5024// a W instruction cheaply.
5025bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
5026 // Look for the sext.w pattern, addiw rd, rs1, 0.
5027 if (N->getMachineOpcode() != RISCV::ADDIW ||
5028 !isNullConstant(V: N->getOperand(Num: 1)))
5029 return false;
5030
5031 SDValue N0 = N->getOperand(Num: 0);
5032 if (!N0.isMachineOpcode())
5033 return false;
5034
5035 switch (N0.getMachineOpcode()) {
5036 default:
5037 break;
5038 case RISCV::ADD:
5039 case RISCV::ADDI:
5040 case RISCV::SUB:
5041 case RISCV::MUL:
5042 case RISCV::SLLI: {
5043 // Convert sext.w+add/sub/mul to their W instructions. This will create
5044 // a new independent instruction. This improves latency.
5045 unsigned Opc;
5046 switch (N0.getMachineOpcode()) {
5047 default:
5048 llvm_unreachable("Unexpected opcode!");
5049 case RISCV::ADD: Opc = RISCV::ADDW; break;
5050 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
5051 case RISCV::SUB: Opc = RISCV::SUBW; break;
5052 case RISCV::MUL: Opc = RISCV::MULW; break;
5053 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
5054 }
5055
5056 SDValue N00 = N0.getOperand(i: 0);
5057 SDValue N01 = N0.getOperand(i: 1);
5058
5059 // Shift amount needs to be uimm5.
5060 if (N0.getMachineOpcode() == RISCV::SLLI &&
5061 !isUInt<5>(x: cast<ConstantSDNode>(Val&: N01)->getSExtValue()))
5062 break;
5063
5064 SDNode *Result =
5065 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VT: N->getValueType(ResNo: 0),
5066 Op1: N00, Op2: N01);
5067 ReplaceUses(F: N, T: Result);
5068 return true;
5069 }
5070 case RISCV::ADDW:
5071 case RISCV::ADDIW:
5072 case RISCV::SUBW:
5073 case RISCV::MULW:
5074 case RISCV::SLLIW:
5075 case RISCV::PACKW:
5076 case RISCV::TH_MULAW:
5077 case RISCV::TH_MULAH:
5078 case RISCV::TH_MULSW:
5079 case RISCV::TH_MULSH:
5080 if (N0.getValueType() == MVT::i32)
5081 break;
5082
5083 // Result is already sign extended just remove the sext.w.
5084 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
5085 ReplaceUses(F: N, T: N0.getNode());
5086 return true;
5087 }
5088
5089 return false;
5090}
5091
5092static bool usesAllOnesMask(SDValue MaskOp) {
5093 const auto IsVMSet = [](unsigned Opc) {
5094 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
5095 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
5096 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
5097 Opc == RISCV::PseudoVMSET_M_B8;
5098 };
5099
5100 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
5101 // undefined behaviour if it's the wrong bitwidth, so we could choose to
5102 // assume that it's all-ones? Same applies to its VL.
5103 return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode());
5104}
5105
5106static bool isImplicitDef(SDValue V) {
5107 if (!V.isMachineOpcode())
5108 return false;
5109 if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
5110 for (unsigned I = 1; I < V.getNumOperands(); I += 2)
5111 if (!isImplicitDef(V: V.getOperand(i: I)))
5112 return false;
5113 return true;
5114 }
5115 return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
5116}
5117
5118// Optimize masked RVV pseudo instructions with a known all-ones mask to their
5119// corresponding "unmasked" pseudo versions.
5120bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
5121 const RISCV::RISCVMaskedPseudoInfo *I =
5122 RISCV::getMaskedPseudoInfo(MaskedPseudo: N->getMachineOpcode());
5123 if (!I)
5124 return false;
5125
5126 unsigned MaskOpIdx = I->MaskOpIdx;
5127 if (!usesAllOnesMask(MaskOp: N->getOperand(Num: MaskOpIdx)))
5128 return false;
5129
5130 // There are two classes of pseudos in the table - compares and
5131 // everything else. See the comment on RISCVMaskedPseudo for details.
5132 const unsigned Opc = I->UnmaskedPseudo;
5133 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
5134 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MCID);
5135
5136 const MCInstrDesc &MaskedMCID = TII->get(Opcode: N->getMachineOpcode());
5137 const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MaskedMCID);
5138
5139 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
5140 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&
5141 "Unmasked pseudo has policy but masked pseudo doesn't?");
5142 assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru &&
5143 "Unexpected pseudo structure");
5144 assert(!(HasPassthru && !MaskedHasPassthru) &&
5145 "Unmasked pseudo has passthru but masked pseudo doesn't?");
5146
5147 SmallVector<SDValue, 8> Ops;
5148 // Skip the passthru operand at index 0 if the unmasked don't have one.
5149 bool ShouldSkip = !HasPassthru && MaskedHasPassthru;
5150 bool DropPolicy = !RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags) &&
5151 RISCVII::hasVecPolicyOp(TSFlags: MaskedMCID.TSFlags);
5152 bool HasChainOp =
5153 N->getOperand(Num: N->getNumOperands() - 1).getValueType() == MVT::Other;
5154 unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp;
5155 for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) {
5156 // Skip the mask
5157 SDValue Op = N->getOperand(Num: I);
5158 if (I == MaskOpIdx)
5159 continue;
5160 if (DropPolicy && I == LastOpNum)
5161 continue;
5162 Ops.push_back(Elt: Op);
5163 }
5164
5165 MachineSDNode *Result =
5166 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5167
5168 if (!N->memoperands_empty())
5169 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: N->memoperands());
5170
5171 Result->setFlags(N->getFlags());
5172 ReplaceUses(F: N, T: Result);
5173
5174 return true;
5175}
5176
5177/// If our passthru is an implicit_def, use noreg instead. This side
5178/// steps issues with MachineCSE not being able to CSE expressions with
5179/// IMPLICIT_DEF operands while preserving the semantic intent. See
5180/// pr64282 for context. Note that this transform is the last one
5181/// performed at ISEL DAG to DAG.
5182bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
5183 bool MadeChange = false;
5184 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5185
5186 while (Position != CurDAG->allnodes_begin()) {
5187 SDNode *N = &*--Position;
5188 if (N->use_empty() || !N->isMachineOpcode())
5189 continue;
5190
5191 const unsigned Opc = N->getMachineOpcode();
5192 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opc) ||
5193 !RISCVII::isFirstDefTiedToFirstUse(Desc: TII->get(Opcode: Opc)) ||
5194 !isImplicitDef(V: N->getOperand(Num: 0)))
5195 continue;
5196
5197 SmallVector<SDValue> Ops;
5198 Ops.push_back(Elt: CurDAG->getRegister(Reg: RISCV::NoRegister, VT: N->getValueType(ResNo: 0)));
5199 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
5200 SDValue Op = N->getOperand(Num: I);
5201 Ops.push_back(Elt: Op);
5202 }
5203
5204 MachineSDNode *Result =
5205 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5206 Result->setFlags(N->getFlags());
5207 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: cast<MachineSDNode>(Val: N)->memoperands());
5208 ReplaceUses(F: N, T: Result);
5209 MadeChange = true;
5210 }
5211 return MadeChange;
5212}
5213
5214
5215// This pass converts a legalized DAG into a RISCV-specific DAG, ready
5216// for instruction scheduling.
5217FunctionPass *llvm::createRISCVISelDagLegacyPass(RISCVTargetMachine &TM,
5218 CodeGenOptLevel OptLevel) {
5219 return new RISCVDAGToDAGISelLegacy(TM, OptLevel);
5220}
5221
5222RISCVISelDAGToDAGPass::RISCVISelDAGToDAGPass(RISCVTargetMachine &TM,
5223 CodeGenOptLevel OptLevel)
5224 : SelectionDAGISelPass(std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5225
5226char RISCVDAGToDAGISelLegacy::ID = 0;
5227
5228RISCVDAGToDAGISelLegacy::RISCVDAGToDAGISelLegacy(RISCVTargetMachine &TM,
5229 CodeGenOptLevel OptLevel)
5230 : SelectionDAGISelLegacy(
5231 ID, std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5232
5233INITIALIZE_PASS(RISCVDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
5234