1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "MCTargetDesc/RISCVMatInt.h"
17#include "RISCVISelLowering.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSelectionDAGInfo.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/Support/Alignment.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
32#define GET_DAGISEL_BODY RISCVDAGToDAGISel
33#include "RISCVGenDAGISel.inc"
34
35void RISCVDAGToDAGISel::PreprocessISelDAG() {
36 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
37
38 bool MadeChange = false;
39 while (Position != CurDAG->allnodes_begin()) {
40 SDNode *N = &*--Position;
41 if (N->use_empty())
42 continue;
43
44 SDValue Result;
45 switch (N->getOpcode()) {
46 case ISD::SPLAT_VECTOR: {
47 if (Subtarget->hasStdExtP())
48 break;
49 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
50 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
51 MVT VT = N->getSimpleValueType(ResNo: 0);
52 unsigned Opc =
53 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
54 SDLoc DL(N);
55 SDValue VL = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
56 SDValue Src = N->getOperand(Num: 0);
57 if (VT.isInteger())
58 Src = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget->getXLenVT(),
59 Operand: N->getOperand(Num: 0));
60 Result = CurDAG->getNode(Opcode: Opc, DL, VT, N1: CurDAG->getUNDEF(VT), N2: Src, N3: VL);
61 break;
62 }
63 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
64 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
65 // load. Done after lowering and combining so that we have a chance to
66 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
67 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
68 MVT VT = N->getSimpleValueType(ResNo: 0);
69 SDValue Passthru = N->getOperand(Num: 0);
70 SDValue Lo = N->getOperand(Num: 1);
71 SDValue Hi = N->getOperand(Num: 2);
72 SDValue VL = N->getOperand(Num: 3);
73 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
74 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
75 "Unexpected VTs!");
76 MachineFunction &MF = CurDAG->getMachineFunction();
77 SDLoc DL(N);
78
79 // Create temporary stack for each expanding node.
80 SDValue StackSlot =
81 CurDAG->CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 8), Alignment: Align(8));
82 int FI = cast<FrameIndexSDNode>(Val: StackSlot.getNode())->getIndex();
83 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
84
85 SDValue Chain = CurDAG->getEntryNode();
86 Lo = CurDAG->getStore(Chain, dl: DL, Val: Lo, Ptr: StackSlot, PtrInfo: MPI, Alignment: Align(8));
87
88 SDValue OffsetSlot =
89 CurDAG->getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL);
90 Hi = CurDAG->getStore(Chain, dl: DL, Val: Hi, Ptr: OffsetSlot, PtrInfo: MPI.getWithOffset(O: 4),
91 Alignment: Align(8));
92
93 Chain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
94
95 SDVTList VTs = CurDAG->getVTList(VTs: {VT, MVT::Other});
96 SDValue IntID =
97 CurDAG->getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: MVT::i64);
98 SDValue Ops[] = {Chain,
99 IntID,
100 Passthru,
101 StackSlot,
102 CurDAG->getRegister(Reg: RISCV::X0, VT: MVT::i64),
103 VL};
104
105 Result = CurDAG->getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
106 MemVT: MVT::i64, PtrInfo: MPI, Alignment: Align(8),
107 Flags: MachineMemOperand::MOLoad);
108 break;
109 }
110 case ISD::FP_EXTEND: {
111 // We only have vector patterns for riscv_fpextend_vl in isel.
112 SDLoc DL(N);
113 MVT VT = N->getSimpleValueType(ResNo: 0);
114 if (!VT.isVector())
115 break;
116 SDValue VLMAX = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
117 SDValue TrueMask = CurDAG->getNode(
118 Opcode: RISCVISD::VMSET_VL, DL, VT: VT.changeVectorElementType(EltVT: MVT::i1), Operand: VLMAX);
119 Result = CurDAG->getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT, N1: N->getOperand(Num: 0),
120 N2: TrueMask, N3: VLMAX);
121 break;
122 }
123 case ISD::ADD: {
124 // Turn (add X, C) into (sub X, -C) when a constant node holding -C
125 // already exists in the DAG, so both share one materialization. Do this
126 // before selection, while both are still ConstantSDNodes: by selection
127 // time -C may already have been selected into instructions.
128 //
129 // ADD is commutative, but getNode canonicalizes constants to the RHS, so
130 // the constant is always operand 1.
131 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
132 if (!N1C)
133 break;
134 MVT VT = N->getSimpleValueType(ResNo: 0);
135 if (VT != Subtarget->getXLenVT())
136 break;
137 int64_t Imm = N1C->getSExtValue();
138 // Only worthwhile for wide constants: values that fit in 32 bits take at
139 // most two instructions to materialize, matching the threshold used by
140 // selectNegImm. Skip INT64_MIN too, whose negation is itself.
141 if (isInt<32>(x: Imm) || Imm == INT64_MIN)
142 break;
143 // A constant is anchored if it has a user other than an ADD, i.e. it is
144 // materialized regardless of this fold. N1C is the (unique) node for Imm,
145 // so the positive side needs no search.
146 auto IsAnchored = [](const SDNode *C) {
147 return any_of(Range: C->users(), P: [](const SDNode *U) {
148 return U->getOpcode() != ISD::ADD;
149 });
150 };
151 // If Imm is materialized anyway, keep the ADD so it reuses Imm; an ADD is
152 // also more compressible than a SUB. This also lets us skip the search
153 // for -Imm below.
154 if (IsAnchored(N1C))
155 break;
156 // Find the (unique) constant node for -Imm, if any.
157 const SDNode *NegC = nullptr;
158 for (const SDNode &Node : CurDAG->allnodes()) {
159 auto *C = dyn_cast<ConstantSDNode>(Val: &Node);
160 if (C && C->getSimpleValueType(ResNo: 0) == VT && C->getSExtValue() == -Imm) {
161 NegC = &Node;
162 break;
163 }
164 }
165 // Reuse is only free if -Imm is already in the DAG.
166 if (!NegC)
167 break;
168 // dyn_cast<ConstantSDNode> also matches TargetConstant, which is encoded
169 // into the instruction rather than materialized, so reusing it would not
170 // remove a materialization. No TargetConstant is this wide (the largest
171 // are intrinsic IDs, which fit in 32 bits), so assert it is a Constant.
172 assert(NegC->getOpcode() == ISD::Constant &&
173 "Unexpected wide TargetConstant");
174 // Pick which of Imm/-Imm should be the surviving constant, so exactly
175 // one of the pair is materialized and any ADDs of the other reuse it:
176 // - if -Imm is materialized anyway, reuse it (rewrite to SUB);
177 // - else keep the cheaper constant, breaking ties towards the positive
178 // value so both ADDs of a C/-C pair agree on the survivor.
179 bool Rewrite;
180 if (IsAnchored(NegC)) {
181 Rewrite = true;
182 } else {
183 int PosCost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
184 /*CompressionCost=*/true);
185 int NegCost =
186 RISCVMatInt::getIntMatCost(Val: APInt(64, -Imm), Size: 64, STI: *Subtarget,
187 /*CompressionCost=*/true);
188 Rewrite = NegCost != PosCost ? NegCost < PosCost : Imm < 0;
189 }
190 if (!Rewrite)
191 break;
192 SDLoc DL(N);
193 // getConstant uniques onto the existing -C node, so it is shared.
194 Result = CurDAG->getNode(Opcode: ISD::SUB, DL, VT, N1: N->getOperand(Num: 0),
195 N2: CurDAG->getConstant(Val: -Imm, DL, VT));
196 break;
197 }
198 }
199
200 if (Result) {
201 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
202 LLVM_DEBUG(N->dump(CurDAG));
203 LLVM_DEBUG(dbgs() << "\nNew: ");
204 LLVM_DEBUG(Result->dump(CurDAG));
205 LLVM_DEBUG(dbgs() << "\n");
206
207 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
208 MadeChange = true;
209 }
210 }
211
212 if (MadeChange)
213 CurDAG->RemoveDeadNodes();
214}
215
216void RISCVDAGToDAGISel::PostprocessISelDAG() {
217 HandleSDNode Dummy(CurDAG->getRoot());
218 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
219
220 bool MadeChange = false;
221 while (Position != CurDAG->allnodes_begin()) {
222 SDNode *N = &*--Position;
223 // Skip dead nodes and any non-machine opcodes.
224 if (N->use_empty() || !N->isMachineOpcode())
225 continue;
226
227 MadeChange |= doPeepholeSExtW(Node: N);
228
229 // FIXME: This is here only because the VMerge transform doesn't
230 // know how to handle masked true inputs. Once that has been moved
231 // to post-ISEL, this can be deleted as well.
232 MadeChange |= doPeepholeMaskedRVV(Node: cast<MachineSDNode>(Val: N));
233 }
234
235 CurDAG->setRoot(Dummy.getValue());
236
237 // After we're done with everything else, convert IMPLICIT_DEF
238 // passthru operands to NoRegister. This is required to workaround
239 // an optimization deficiency in MachineCSE. This really should
240 // be merged back into each of the patterns (i.e. there's no good
241 // reason not to go directly to NoReg), but is being done this way
242 // to allow easy backporting.
243 MadeChange |= doPeepholeNoRegPassThru();
244
245 if (MadeChange)
246 CurDAG->RemoveDeadNodes();
247}
248
249static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
250 RISCVMatInt::InstSeq &Seq) {
251 SDValue SrcReg = CurDAG->getRegister(Reg: RISCV::X0, VT);
252 for (const RISCVMatInt::Inst &Inst : Seq) {
253 SDValue SDImm = CurDAG->getSignedTargetConstant(Val: Inst.getImm(), DL, VT);
254 SDNode *Result = nullptr;
255 switch (Inst.getOpndKind()) {
256 case RISCVMatInt::Imm:
257 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SDImm);
258 break;
259 case RISCVMatInt::RegX0:
260 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg,
261 Op2: CurDAG->getRegister(Reg: RISCV::X0, VT));
262 break;
263 case RISCVMatInt::RegReg:
264 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SrcReg);
265 break;
266 case RISCVMatInt::RegImm:
267 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SDImm);
268 break;
269 }
270
271 // Only the first instruction has X0 as its source.
272 SrcReg = SDValue(Result, 0);
273 }
274
275 return SrcReg;
276}
277
278static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
279 int64_t Imm, const RISCVSubtarget &Subtarget) {
280 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
281
282 // Use a rematerializable pseudo instruction for short sequences if enabled.
283 if (Seq.size() == 2 && Subtarget.getCLOpts().use_rematerializable_movimm)
284 return SDValue(
285 CurDAG->getMachineNode(Opcode: RISCV::PseudoMovImm, dl: DL, VT,
286 Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT)),
287 0);
288
289 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
290 // worst an LUI+ADDIW. This will require an extra register, but avoids a
291 // constant pool.
292 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
293 // low and high 32 bits are the same and bit 31 and 63 are set.
294 if (Seq.size() > 3) {
295 unsigned ShiftAmt, AddOpc;
296 RISCVMatInt::InstSeq SeqLo =
297 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
298 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
299 SDValue Lo = selectImmSeq(CurDAG, DL, VT, Seq&: SeqLo);
300
301 SDValue SLLI = SDValue(
302 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: Lo,
303 Op2: CurDAG->getTargetConstant(Val: ShiftAmt, DL, VT)),
304 0);
305 return SDValue(CurDAG->getMachineNode(Opcode: AddOpc, dl: DL, VT, Op1: Lo, Op2: SLLI), 0);
306 }
307 }
308
309 // Otherwise, use the original sequence.
310 return selectImmSeq(CurDAG, DL, VT, Seq);
311}
312
313void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
314 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
315 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
316 bool IsLoad, MVT *IndexVT) {
317 SDValue Chain = Node->getOperand(Num: 0);
318
319 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Base pointer.
320
321 if (IsStridedOrIndexed) {
322 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Index.
323 if (IndexVT)
324 *IndexVT = Operands.back()->getSimpleValueType(ResNo: 0);
325 }
326
327 if (IsMasked) {
328 SDValue Mask = Node->getOperand(Num: CurOp++);
329 Operands.push_back(Elt: Mask);
330 }
331 SDValue VL;
332 selectVLOp(N: Node->getOperand(Num: CurOp++), VL);
333 Operands.push_back(Elt: VL);
334
335 MVT XLenVT = Subtarget->getXLenVT();
336 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
337 Operands.push_back(Elt: SEWOp);
338
339 // At the IR layer, all the masked load intrinsics have policy operands,
340 // none of the others do. All have passthru operands. For our pseudos,
341 // all loads have policy operands.
342 if (IsLoad) {
343 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
344 if (IsMasked)
345 Policy = Node->getConstantOperandVal(Num: CurOp++);
346 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
347 Operands.push_back(Elt: PolicyOp);
348 }
349
350 Operands.push_back(Elt: Chain); // Chain.
351}
352
353void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked,
354 bool IsStrided) {
355 SDLoc DL(Node);
356 MVT VT = Node->getSimpleValueType(ResNo: 0);
357 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
358 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
359
360 unsigned CurOp = 2;
361 SmallVector<SDValue, 8> Operands;
362
363 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
364
365 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
366 Operands, /*IsLoad=*/true);
367
368 const RISCV::VLSEGPseudo *P =
369 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
370 LMUL: static_cast<unsigned>(LMUL));
371 MachineSDNode *Load =
372 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
373
374 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
375
376 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
377 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
378 CurDAG->RemoveDeadNode(N: Node);
379}
380
381void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, unsigned NF,
382 bool IsMasked) {
383 SDLoc DL(Node);
384 MVT VT = Node->getSimpleValueType(ResNo: 0);
385 MVT XLenVT = Subtarget->getXLenVT();
386 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
387 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
388
389 unsigned CurOp = 2;
390 SmallVector<SDValue, 7> Operands;
391
392 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
393
394 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
395 /*IsStridedOrIndexed*/ false, Operands,
396 /*IsLoad=*/true);
397
398 const RISCV::VLSEGPseudo *P =
399 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
400 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
401 MachineSDNode *Load = CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped,
402 VT2: XLenVT, VT3: MVT::Other, Ops: Operands);
403
404 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
405
406 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0)); // Result
407 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1)); // VL
408 ReplaceUses(F: SDValue(Node, 2), T: SDValue(Load, 2)); // Chain
409 CurDAG->RemoveDeadNode(N: Node);
410}
411
412void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked,
413 bool IsOrdered) {
414 SDLoc DL(Node);
415 MVT VT = Node->getSimpleValueType(ResNo: 0);
416 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
417 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
418
419 unsigned CurOp = 2;
420 SmallVector<SDValue, 8> Operands;
421
422 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
423
424 MVT IndexVT;
425 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
426 /*IsStridedOrIndexed*/ true, Operands,
427 /*IsLoad=*/true, IndexVT: &IndexVT);
428
429#ifndef NDEBUG
430 // Number of element = RVVBitsPerBlock * LMUL / SEW
431 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
432 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
433 if (DecodedLMUL.second)
434 ContainedTyNumElts /= DecodedLMUL.first;
435 else
436 ContainedTyNumElts *= DecodedLMUL.first;
437 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
438 "Element count mismatch");
439#endif
440
441 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
442 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
443 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
444 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
445 "values when XLEN=32");
446 }
447 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
448 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
449 IndexLMUL: static_cast<unsigned>(IndexLMUL));
450 MachineSDNode *Load =
451 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
452
453 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
454
455 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
456 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
457 CurDAG->RemoveDeadNode(N: Node);
458}
459
460void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked,
461 bool IsStrided) {
462 SDLoc DL(Node);
463 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
464 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
465 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
466
467 unsigned CurOp = 2;
468 SmallVector<SDValue, 8> Operands;
469
470 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
471
472 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
473 Operands);
474
475 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
476 NF, Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
477 MachineSDNode *Store =
478 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
479
480 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
481
482 ReplaceNode(F: Node, T: Store);
483}
484
485void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked,
486 bool IsOrdered) {
487 SDLoc DL(Node);
488 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
489 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
490 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
491
492 unsigned CurOp = 2;
493 SmallVector<SDValue, 8> Operands;
494
495 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
496
497 MVT IndexVT;
498 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
499 /*IsStridedOrIndexed*/ true, Operands,
500 /*IsLoad=*/false, IndexVT: &IndexVT);
501
502#ifndef NDEBUG
503 // Number of element = RVVBitsPerBlock * LMUL / SEW
504 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
505 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
506 if (DecodedLMUL.second)
507 ContainedTyNumElts /= DecodedLMUL.first;
508 else
509 ContainedTyNumElts *= DecodedLMUL.first;
510 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
511 "Element count mismatch");
512#endif
513
514 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
515 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
516 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
517 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
518 "values when XLEN=32");
519 }
520 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
521 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
522 IndexLMUL: static_cast<unsigned>(IndexLMUL));
523 MachineSDNode *Store =
524 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
525
526 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
527
528 ReplaceNode(F: Node, T: Store);
529}
530
531void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) {
532 if (!Subtarget->hasVInstructions())
533 return;
534
535 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
536
537 SDLoc DL(Node);
538 MVT XLenVT = Subtarget->getXLenVT();
539
540 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
541
542 assert((IntNo == Intrinsic::riscv_vsetvli ||
543 IntNo == Intrinsic::riscv_vsetvlimax) &&
544 "Unexpected vsetvli intrinsic");
545
546 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
547 unsigned Offset = (VLMax ? 1 : 2);
548
549 assert(Node->getNumOperands() == Offset + 2 &&
550 "Unexpected number of operands");
551
552 unsigned SEW =
553 RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: Offset) & 0x7);
554 RISCVVType::VLMUL VLMul = static_cast<RISCVVType::VLMUL>(
555 Node->getConstantOperandVal(Num: Offset + 1) & 0x7);
556
557 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
558 /*MaskAgnostic*/ true);
559 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
560
561 SDValue VLOperand;
562 unsigned Opcode = RISCV::PseudoVSETVLI;
563 if (auto *C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
564 if (auto VLEN = Subtarget->getRealVLen())
565 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
566 VLMax = true;
567 }
568 if (VLMax || isAllOnesConstant(V: Node->getOperand(Num: 1))) {
569 VLOperand = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
570 Opcode = RISCV::PseudoVSETVLIX0;
571 } else {
572 VLOperand = Node->getOperand(Num: 1);
573
574 if (auto *C = dyn_cast<ConstantSDNode>(Val&: VLOperand)) {
575 uint64_t AVL = C->getZExtValue();
576 if (isUInt<5>(x: AVL)) {
577 SDValue VLImm = CurDAG->getTargetConstant(Val: AVL, DL, VT: XLenVT);
578 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: RISCV::PseudoVSETIVLI, dl: DL,
579 VT: XLenVT, Op1: VLImm, Op2: VTypeIOp));
580 return;
581 }
582 }
583 }
584
585 ReplaceNode(F: Node,
586 T: CurDAG->getMachineNode(Opcode, dl: DL, VT: XLenVT, Op1: VLOperand, Op2: VTypeIOp));
587}
588
589void RISCVDAGToDAGISel::selectXSfmmVSET(SDNode *Node) {
590 if (!Subtarget->hasVendorXSfmmbase())
591 return;
592
593 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
594
595 SDLoc DL(Node);
596 MVT XLenVT = Subtarget->getXLenVT();
597
598 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
599
600 assert((IntNo == Intrinsic::riscv_sf_vsettnt ||
601 IntNo == Intrinsic::riscv_sf_vsettm ||
602 IntNo == Intrinsic::riscv_sf_vsettk) &&
603 "Unexpected XSfmm vset intrinsic");
604
605 unsigned SEW = RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: 2));
606 unsigned Widen = RISCVVType::decodeTWiden(TWiden: Node->getConstantOperandVal(Num: 3));
607 unsigned PseudoOpCode =
608 IntNo == Intrinsic::riscv_sf_vsettnt ? RISCV::PseudoSF_VSETTNT
609 : IntNo == Intrinsic::riscv_sf_vsettm ? RISCV::PseudoSF_VSETTM
610 : RISCV::PseudoSF_VSETTK;
611
612 if (IntNo == Intrinsic::riscv_sf_vsettnt) {
613 unsigned VTypeI = RISCVVType::encodeXSfmmVType(SEW, Widen, AltFmt: 0);
614 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
615
616 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
617 Op1: Node->getOperand(Num: 1), Op2: VTypeIOp));
618 } else {
619 SDValue Log2SEW = CurDAG->getTargetConstant(Val: Log2_32(Value: SEW), DL, VT: XLenVT);
620 SDValue TWiden = CurDAG->getTargetConstant(Val: Widen, DL, VT: XLenVT);
621 ReplaceNode(F: Node,
622 T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
623 Op1: Node->getOperand(Num: 1), Op2: Log2SEW, Op3: TWiden));
624 }
625}
626
627bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
628 MVT VT = Node->getSimpleValueType(ResNo: 0);
629 unsigned Opcode = Node->getOpcode();
630 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
631 "Unexpected opcode");
632 SDLoc DL(Node);
633
634 // For operations of the form (x << C1) op C2, check if we can use
635 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
636 SDValue N0 = Node->getOperand(Num: 0);
637 SDValue N1 = Node->getOperand(Num: 1);
638
639 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
640 if (!Cst)
641 return false;
642
643 int64_t Val = Cst->getSExtValue();
644
645 // Check if immediate can already use ANDI/ORI/XORI.
646 if (isInt<12>(x: Val))
647 return false;
648
649 SDValue Shift = N0;
650
651 // If Val is simm32 and we have a sext_inreg from i32, then the binop
652 // produces at least 33 sign bits. We can peek through the sext_inreg and use
653 // a SLLIW at the end.
654 bool SignExt = false;
655 if (isInt<32>(x: Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
656 N0.hasOneUse() && cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT() == MVT::i32) {
657 SignExt = true;
658 Shift = N0.getOperand(i: 0);
659 }
660
661 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
662 return false;
663
664 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1));
665 if (!ShlCst)
666 return false;
667
668 uint64_t ShAmt = ShlCst->getZExtValue();
669 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
670 "Unexpected shift amount!");
671
672 // Make sure that we don't change the operation by removing bits.
673 // This only matters for OR and XOR, AND is unaffected.
674 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(N: ShAmt);
675 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
676 return false;
677
678 int64_t ShiftedVal = Val >> ShAmt;
679 if (!isInt<12>(x: ShiftedVal))
680 return false;
681
682 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
683 if (SignExt && ShAmt >= 32)
684 return false;
685
686 // Ok, we can reorder to get a smaller immediate.
687 unsigned BinOpc;
688 switch (Opcode) {
689 default: llvm_unreachable("Unexpected opcode");
690 case ISD::AND: BinOpc = RISCV::ANDI; break;
691 case ISD::OR: BinOpc = RISCV::ORI; break;
692 case ISD::XOR: BinOpc = RISCV::XORI; break;
693 }
694
695 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
696
697 SDNode *BinOp = CurDAG->getMachineNode(
698 Opcode: BinOpc, dl: DL, VT, Op1: Shift.getOperand(i: 0),
699 Op2: CurDAG->getSignedTargetConstant(Val: ShiftedVal, DL, VT));
700 SDNode *SLLI =
701 CurDAG->getMachineNode(Opcode: ShOpc, dl: DL, VT, Op1: SDValue(BinOp, 0),
702 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
703 ReplaceNode(F: Node, T: SLLI);
704 return true;
705}
706
707bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
708 unsigned Opc;
709
710 if (Subtarget->hasVendorXTHeadBb())
711 Opc = RISCV::TH_EXT;
712 else if (Subtarget->hasVendorXAndesPerf())
713 Opc = RISCV::NDS_BFOS;
714 else if (Subtarget->hasVendorXqcibm())
715 Opc = RISCV::QC_EXT;
716 else
717 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
718 return false;
719
720 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
721 if (!N1C)
722 return false;
723
724 SDValue N0 = Node->getOperand(Num: 0);
725 if (!N0.hasOneUse())
726 return false;
727
728 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
729 const SDLoc &DL, MVT VT) {
730 if (Opc == RISCV::QC_EXT) {
731 // QC.EXT X, width, shamt
732 // shamt is the same as Lsb
733 // width is the number of bits to extract from the Lsb
734 Msb = Msb - Lsb + 1;
735 }
736 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
737 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
738 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
739 };
740
741 SDLoc DL(Node);
742 MVT VT = Node->getSimpleValueType(ResNo: 0);
743 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
744 const unsigned RightShAmt = N1C->getZExtValue();
745 assert(RightShAmt > 0 && RightShAmt < VT.getSizeInBits() &&
746 "Unexpected shift amount!");
747
748 // Transform (sra (shl X, C1) C2) with C1 < C2
749 // -> (SignedBitfieldExtract X, msb, lsb)
750 if (N0.getOpcode() == ISD::SHL) {
751 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
752 if (!N01C)
753 return false;
754
755 const unsigned LeftShAmt = N01C->getZExtValue();
756 assert(LeftShAmt > 0 && LeftShAmt < VT.getSizeInBits() &&
757 "Unexpected shift amount!");
758 // Make sure that this is a bitfield extraction (i.e., the shift-right
759 // amount can not be less than the left-shift).
760 if (LeftShAmt > RightShAmt)
761 return false;
762
763 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
764 const unsigned Msb = MsbPlusOne - 1;
765 const unsigned Lsb = RightShAmt - LeftShAmt;
766
767 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
768 ReplaceNode(F: Node, T: Sbe);
769 return true;
770 }
771
772 // Transform (sra (sext_inreg X, _), C) ->
773 // (SignedBitfieldExtract X, msb, lsb)
774 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
775 unsigned ExtSize =
776 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
777
778 // ExtSize of 32 should use sraiw via tablegen pattern.
779 if (ExtSize == 32)
780 return false;
781
782 const unsigned Msb = ExtSize - 1;
783 // If the shift-right amount is greater than Msb, it means that extracts
784 // the X[Msb] bit and sign-extend it.
785 const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
786
787 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
788 ReplaceNode(F: Node, T: Sbe);
789 return true;
790 }
791
792 return false;
793}
794
795bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
796 // Only supported with XAndesPerf at the moment.
797 if (!Subtarget->hasVendorXAndesPerf())
798 return false;
799
800 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
801 if (!N1C)
802 return false;
803
804 SDValue N0 = Node->getOperand(Num: 0);
805 if (!N0.hasOneUse())
806 return false;
807
808 auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
809 const SDLoc &DL, MVT VT) {
810 unsigned Opc = RISCV::NDS_BFOS;
811 // If the Lsb is equal to the Msb, then the Lsb should be 0.
812 if (Lsb == Msb)
813 Lsb = 0;
814 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
815 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
816 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
817 };
818
819 SDLoc DL(Node);
820 MVT VT = Node->getSimpleValueType(ResNo: 0);
821 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
822 const unsigned RightShAmt = N1C->getZExtValue();
823 assert(RightShAmt > 0 && RightShAmt < VT.getSizeInBits() &&
824 "Unexpected shift amount!");
825
826 // Transform (sra (shl X, C1) C2) with C1 > C2
827 // -> (NDS.BFOS X, lsb, msb)
828 if (N0.getOpcode() == ISD::SHL) {
829 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
830 if (!N01C)
831 return false;
832
833 const unsigned LeftShAmt = N01C->getZExtValue();
834 assert(LeftShAmt > 0 && LeftShAmt < VT.getSizeInBits() &&
835 "Unexpected shift amount!");
836 // Make sure that this is a bitfield insertion (i.e., the shift-right
837 // amount should be less than the left-shift).
838 if (LeftShAmt <= RightShAmt)
839 return false;
840
841 const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
842 const unsigned Msb = MsbPlusOne - 1;
843 const unsigned Lsb = LeftShAmt - RightShAmt;
844
845 SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
846 ReplaceNode(F: Node, T: Sbi);
847 return true;
848 }
849
850 return false;
851}
852
853bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
854 const SDLoc &DL, MVT VT,
855 SDValue X, unsigned Msb,
856 unsigned Lsb) {
857 unsigned Opc;
858
859 if (Subtarget->hasVendorXTHeadBb()) {
860 Opc = RISCV::TH_EXTU;
861 } else if (Subtarget->hasVendorXAndesPerf()) {
862 Opc = RISCV::NDS_BFOZ;
863 } else if (Subtarget->hasVendorXqcibm()) {
864 Opc = RISCV::QC_EXTU;
865 // QC.EXTU X, width, shamt
866 // shamt is the same as Lsb
867 // width is the number of bits to extract from the Lsb
868 Msb = Msb - Lsb + 1;
869 } else {
870 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
871 return false;
872 }
873
874 SDNode *Ube = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
875 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
876 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
877 ReplaceNode(F: Node, T: Ube);
878 return true;
879}
880
881bool RISCVDAGToDAGISel::tryUnsignedBitfieldInsertInZero(SDNode *Node,
882 const SDLoc &DL, MVT VT,
883 SDValue X, unsigned Msb,
884 unsigned Lsb) {
885 // Only supported with XAndesPerf at the moment.
886 if (!Subtarget->hasVendorXAndesPerf())
887 return false;
888
889 unsigned Opc = RISCV::NDS_BFOZ;
890
891 // If the Lsb is equal to the Msb, then the Lsb should be 0.
892 if (Lsb == Msb)
893 Lsb = 0;
894 SDNode *Ubi = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
895 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
896 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
897 ReplaceNode(F: Node, T: Ubi);
898 return true;
899}
900
901bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
902 // Target does not support indexed loads.
903 if (!Subtarget->hasVendorXTHeadMemIdx())
904 return false;
905
906 LoadSDNode *Ld = cast<LoadSDNode>(Val: Node);
907 ISD::MemIndexedMode AM = Ld->getAddressingMode();
908 if (AM == ISD::UNINDEXED)
909 return false;
910
911 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Ld->getOffset());
912 if (!C)
913 return false;
914
915 EVT LoadVT = Ld->getMemoryVT();
916 assert((AM == ISD::PRE_INC || AM == ISD::POST_INC) &&
917 "Unexpected addressing mode");
918 bool IsPre = AM == ISD::PRE_INC;
919 bool IsPost = AM == ISD::POST_INC;
920 int64_t Offset = C->getSExtValue();
921
922 // The constants that can be encoded in the THeadMemIdx instructions
923 // are of the form (sign_extend(imm5) << imm2).
924 unsigned Shift;
925 for (Shift = 0; Shift < 4; Shift++)
926 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
927 break;
928
929 // Constant cannot be encoded.
930 if (Shift == 4)
931 return false;
932
933 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
934 unsigned Opcode;
935 if (LoadVT == MVT::i8 && IsPre)
936 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
937 else if (LoadVT == MVT::i8 && IsPost)
938 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
939 else if (LoadVT == MVT::i16 && IsPre)
940 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
941 else if (LoadVT == MVT::i16 && IsPost)
942 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
943 else if (LoadVT == MVT::i32 && IsPre)
944 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
945 else if (LoadVT == MVT::i32 && IsPost)
946 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
947 else if (LoadVT == MVT::i64 && IsPre)
948 Opcode = RISCV::TH_LDIB;
949 else if (LoadVT == MVT::i64 && IsPost)
950 Opcode = RISCV::TH_LDIA;
951 else
952 return false;
953
954 EVT Ty = Ld->getOffset().getValueType();
955 SDValue Ops[] = {
956 Ld->getBasePtr(),
957 CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(Node), VT: Ty),
958 CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(Node), VT: Ty), Ld->getChain()};
959 SDNode *New = CurDAG->getMachineNode(Opcode, dl: SDLoc(Node), VT1: Ld->getValueType(ResNo: 0),
960 VT2: Ld->getValueType(ResNo: 1), VT3: MVT::Other, Ops);
961
962 MachineMemOperand *MemOp = cast<MemSDNode>(Val: Node)->getMemOperand();
963 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: New), NewMemRefs: {MemOp});
964
965 ReplaceNode(F: Node, T: New);
966
967 return true;
968}
969
970static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT,
971 SDValue Lo, SDValue Hi) {
972 SDValue Ops[] = {
973 CurDAG->getTargetConstant(Val: RISCV::GPRPairNoX0RegClassID, DL, VT: MVT::i32), Lo,
974 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_even, DL, VT: MVT::i32), Hi,
975 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_odd, DL, VT: MVT::i32)};
976
977 return SDValue(
978 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT, Ops), 0);
979}
980
981// Helper to extract Lo and Hi values from a GPR pair.
982static std::pair<SDValue, SDValue>
983extractGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, SDValue Pair) {
984 SDValue Lo =
985 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL, VT: MVT::i32, Operand: Pair);
986 SDValue Hi =
987 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_odd, DL, VT: MVT::i32, Operand: Pair);
988 return {Lo, Hi};
989}
990
991// Try to match WMACC pattern: ADDD where one operand pair comes from a
992// widening multiply (both results of UMUL_LOHI, SMUL_LOHI, or WMULSU).
993bool RISCVDAGToDAGISel::tryWideningMulAcc(SDNode *Node, const SDLoc &DL) {
994 assert(Node->getOpcode() == RISCVISD::ADDD && "Expected ADDD");
995
996 SDValue Op0Lo = Node->getOperand(Num: 0);
997 SDValue Op0Hi = Node->getOperand(Num: 1);
998 SDValue Op1Lo = Node->getOperand(Num: 2);
999 SDValue Op1Hi = Node->getOperand(Num: 3);
1000
1001 auto IsSupportedMulWithOneUse = [](SDValue Lo, SDValue Hi) {
1002 unsigned Opc = Lo.getOpcode();
1003 if (Opc != ISD::UMUL_LOHI && Opc != ISD::SMUL_LOHI &&
1004 Opc != RISCVISD::WMULSU)
1005 return false;
1006 return Lo.getNode() == Hi.getNode() && Lo.getResNo() == 0 &&
1007 Hi.getResNo() == 1 && Lo.hasOneUse() && Hi.hasOneUse();
1008 };
1009
1010 SDNode *MulNode = nullptr;
1011 SDValue AddLo, AddHi;
1012
1013 // Check if first operand pair is a supported multiply with single use.
1014 if (IsSupportedMulWithOneUse(Op0Lo, Op0Hi)) {
1015 MulNode = Op0Lo.getNode();
1016 AddLo = Op1Lo;
1017 AddHi = Op1Hi;
1018 }
1019 // ADDD is commutative. Check if second operand pair is a supported multiply
1020 // with single use.
1021 else if (IsSupportedMulWithOneUse(Op1Lo, Op1Hi)) {
1022 MulNode = Op1Lo.getNode();
1023 AddLo = Op0Lo;
1024 AddHi = Op0Hi;
1025 } else {
1026 return false;
1027 }
1028
1029 unsigned Opc;
1030 switch (MulNode->getOpcode()) {
1031 default:
1032 llvm_unreachable("Unexpected multiply opcode");
1033 case ISD::UMUL_LOHI:
1034 Opc = RISCV::WMACCU;
1035 break;
1036 case ISD::SMUL_LOHI:
1037 Opc = RISCV::WMACC;
1038 break;
1039 case RISCVISD::WMULSU:
1040 Opc = RISCV::WMACCSU;
1041 break;
1042 }
1043
1044 SDValue Acc = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: AddLo, Hi: AddHi);
1045
1046 // WMACC instruction format: rd, rs1, rs2 (rd is accumulator).
1047 SDValue M0 = MulNode->getOperand(Num: 0);
1048 SDValue M1 = MulNode->getOperand(Num: 1);
1049 MachineSDNode *New =
1050 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Acc, Op2: M0, Op3: M1);
1051
1052 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
1053 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1054 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1055 CurDAG->RemoveDeadNode(N: Node);
1056 return true;
1057}
1058
1059static Register getTileReg(uint64_t TileNum) {
1060 assert(TileNum <= 15 && "Invalid tile number");
1061 return RISCV::T0 + TileNum;
1062}
1063
1064void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
1065 if (!Subtarget->hasVInstructions())
1066 return;
1067
1068 assert(Node->getOpcode() == ISD::INTRINSIC_VOID && "Unexpected opcode");
1069
1070 SDLoc DL(Node);
1071 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
1072
1073 assert((IntNo == Intrinsic::riscv_sf_vc_x_se ||
1074 IntNo == Intrinsic::riscv_sf_vc_i_se) &&
1075 "Unexpected vsetvli intrinsic");
1076
1077 // imm, imm, imm, simm5/scalar, sew, log2lmul, vl
1078 unsigned Log2SEW = Log2_32(Value: Node->getConstantOperandVal(Num: 6));
1079 SDValue SEWOp =
1080 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: Subtarget->getXLenVT());
1081 SmallVector<SDValue, 8> Operands = {Node->getOperand(Num: 2), Node->getOperand(Num: 3),
1082 Node->getOperand(Num: 4), Node->getOperand(Num: 5),
1083 Node->getOperand(Num: 8), SEWOp,
1084 Node->getOperand(Num: 0)};
1085
1086 unsigned Opcode;
1087 auto *LMulSDNode = cast<ConstantSDNode>(Val: Node->getOperand(Num: 7));
1088 switch (LMulSDNode->getSExtValue()) {
1089 case 5:
1090 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
1091 : RISCV::PseudoSF_VC_I_SE_MF8;
1092 break;
1093 case 6:
1094 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
1095 : RISCV::PseudoSF_VC_I_SE_MF4;
1096 break;
1097 case 7:
1098 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
1099 : RISCV::PseudoSF_VC_I_SE_MF2;
1100 break;
1101 case 0:
1102 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
1103 : RISCV::PseudoSF_VC_I_SE_M1;
1104 break;
1105 case 1:
1106 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
1107 : RISCV::PseudoSF_VC_I_SE_M2;
1108 break;
1109 case 2:
1110 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
1111 : RISCV::PseudoSF_VC_I_SE_M4;
1112 break;
1113 case 3:
1114 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
1115 : RISCV::PseudoSF_VC_I_SE_M8;
1116 break;
1117 }
1118
1119 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
1120 Opcode, dl: DL, VT: Node->getSimpleValueType(ResNo: 0), Ops: Operands));
1121}
1122
1123static unsigned getSegInstNF(unsigned Intrinsic) {
1124#define INST_NF_CASE(NAME, NF) \
1125 case Intrinsic::riscv_##NAME##NF: \
1126 return NF;
1127#define INST_NF_CASE_MASK(NAME, NF) \
1128 case Intrinsic::riscv_##NAME##NF##_mask: \
1129 return NF;
1130#define INST_NF_CASE_FF(NAME, NF) \
1131 case Intrinsic::riscv_##NAME##NF##ff: \
1132 return NF;
1133#define INST_NF_CASE_FF_MASK(NAME, NF) \
1134 case Intrinsic::riscv_##NAME##NF##ff_mask: \
1135 return NF;
1136#define INST_ALL_NF_CASE_BASE(MACRO_NAME, NAME) \
1137 MACRO_NAME(NAME, 2) \
1138 MACRO_NAME(NAME, 3) \
1139 MACRO_NAME(NAME, 4) \
1140 MACRO_NAME(NAME, 5) \
1141 MACRO_NAME(NAME, 6) \
1142 MACRO_NAME(NAME, 7) \
1143 MACRO_NAME(NAME, 8)
1144#define INST_ALL_NF_CASE(NAME) \
1145 INST_ALL_NF_CASE_BASE(INST_NF_CASE, NAME) \
1146 INST_ALL_NF_CASE_BASE(INST_NF_CASE_MASK, NAME)
1147#define INST_ALL_NF_CASE_WITH_FF(NAME) \
1148 INST_ALL_NF_CASE(NAME) \
1149 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF, NAME) \
1150 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF_MASK, NAME)
1151 switch (Intrinsic) {
1152 default:
1153 llvm_unreachable("Unexpected segment load/store intrinsic");
1154 INST_ALL_NF_CASE_WITH_FF(vlseg)
1155 INST_ALL_NF_CASE(vlsseg)
1156 INST_ALL_NF_CASE(vloxseg)
1157 INST_ALL_NF_CASE(vluxseg)
1158 INST_ALL_NF_CASE(vsseg)
1159 INST_ALL_NF_CASE(vssseg)
1160 INST_ALL_NF_CASE(vsoxseg)
1161 INST_ALL_NF_CASE(vsuxseg)
1162 }
1163}
1164
1165static bool isApplicableToPLIOrPLUI(int Val) {
1166 // Check if the immediate is packed i8 or i10
1167 int16_t Bit31To16 = Val >> 16;
1168 int16_t Bit15To0 = Val;
1169 int8_t Bit15To8 = Bit15To0 >> 8;
1170 int8_t Bit7To0 = Val;
1171 if (Bit31To16 != Bit15To0)
1172 return false;
1173
1174 return isInt<10>(x: Bit15To0) || isShiftedInt<10, 6>(x: Bit15To0) ||
1175 Bit15To8 == Bit7To0;
1176}
1177
1178void RISCVDAGToDAGISel::Select(SDNode *Node) {
1179 // If we have a custom node, we have already selected.
1180 if (Node->isMachineOpcode()) {
1181 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
1182 Node->setNodeId(-1);
1183 return;
1184 }
1185
1186 // Instruction Selection not handled by the auto-generated tablegen selection
1187 // should be handled here.
1188 unsigned Opcode = Node->getOpcode();
1189 MVT XLenVT = Subtarget->getXLenVT();
1190 SDLoc DL(Node);
1191 MVT VT = Node->getSimpleValueType(ResNo: 0);
1192
1193 bool HasBitTest = Subtarget->hasBEXTILike();
1194
1195 switch (Opcode) {
1196 case ISD::Constant: {
1197 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1198 auto *ConstNode = cast<ConstantSDNode>(Val: Node);
1199 if (ConstNode->isZero()) {
1200 SDValue New =
1201 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: RISCV::X0, VT);
1202 ReplaceNode(F: Node, T: New.getNode());
1203 return;
1204 }
1205 int64_t Imm = ConstNode->getSExtValue();
1206 // If only the lower 8 bits are used, try to convert this to a simm6 by
1207 // sign-extending bit 7. This is neutral without the C extension, and
1208 // allows C.LI to be used if C is present.
1209 if (!isInt<8>(x: Imm) && isUInt<8>(x: Imm) && isInt<6>(x: SignExtend64<8>(x: Imm)) &&
1210 hasAllBUsers(Node))
1211 Imm = SignExtend64<8>(x: Imm);
1212 // If the upper XLen-16 bits are not used, try to convert this to a simm12
1213 // by sign extending bit 15.
1214 else if (!isInt<16>(x: Imm) && isUInt<16>(x: Imm) &&
1215 isInt<12>(x: SignExtend64<16>(x: Imm)) && hasAllHUsers(Node))
1216 Imm = SignExtend64<16>(x: Imm);
1217
1218 // If the upper XLen-16 bits are not used, the lower 2 bytes are the same,
1219 // and we can't use li, convert to an xlen splat so we can use pli.b.
1220 if (Subtarget->hasStdExtP() && !isInt<12>(x: Imm) &&
1221 (Imm & 0xff) == ((Imm >> 8) & 0xff) && hasAllHUsers(Node)) {
1222 // Splat the lower 16 bits to XLen. Sign extend for RV32.
1223 uint64_t Splat = Imm & 0xffff;
1224 Splat = (Splat << 16) | Splat;
1225 if (VT == MVT::i64)
1226 Imm = Splat << 32 | Splat;
1227 else
1228 Imm = SignExtend64<32>(x: Splat);
1229 } else {
1230 // If the upper 32-bits are not used try to convert this into a simm32 by
1231 // sign extending bit 32.
1232 if (!isInt<32>(x: Imm) && isUInt<32>(x: Imm) && hasAllWUsers(Node))
1233 Imm = SignExtend64<32>(x: Imm);
1234
1235 if (VT == MVT::i64 && !isInt<12>(x: Imm) && !isShiftedInt<20, 12>(x: Imm) &&
1236 Subtarget->hasStdExtP() && isApplicableToPLIOrPLUI(Val: Imm) &&
1237 hasAllWUsers(Node)) {
1238 // If it's 4 packed 8-bit integers or 2 packed signed 16-bit integers,
1239 // we can simply copy lower 32 bits to higher 32 bits to make it able to
1240 // rematerialize to PLI_B or PLI_H
1241 Imm = ((uint64_t)Imm << 32) | (Imm & 0xFFFFFFFF);
1242 }
1243 }
1244
1245 ReplaceNode(F: Node, T: selectImm(CurDAG, DL, VT, Imm, Subtarget: *Subtarget).getNode());
1246 return;
1247 }
1248 case ISD::ConstantFP: {
1249 const APFloat &APF = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1250
1251 bool Is64Bit = Subtarget->is64Bit();
1252 bool HasZdinx = Subtarget->hasStdExtZdinx();
1253
1254 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
1255 SDValue Imm;
1256 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
1257 // create an integer immediate.
1258 if (APF.isPosZero() || NegZeroF64) {
1259 if (VT == MVT::f64 && HasZdinx && !Is64Bit)
1260 Imm = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::f64);
1261 else
1262 Imm = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
1263 } else {
1264 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
1265 Subtarget: *Subtarget);
1266 }
1267
1268 unsigned Opc;
1269 switch (VT.SimpleTy) {
1270 default:
1271 llvm_unreachable("Unexpected size");
1272 case MVT::bf16:
1273 assert(Subtarget->hasStdExtZfbfmin());
1274 Opc = RISCV::FMV_H_X;
1275 break;
1276 case MVT::f16:
1277 Opc = Subtarget->hasStdExtZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
1278 break;
1279 case MVT::f32:
1280 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
1281 break;
1282 case MVT::f64:
1283 // For RV32, we can't move from a GPR, we need to convert instead. This
1284 // should only happen for +0.0 and -0.0.
1285 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
1286 if (HasZdinx)
1287 Opc = RISCV::COPY;
1288 else
1289 Opc = Is64Bit ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
1290 break;
1291 }
1292
1293 SDNode *Res;
1294 if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
1295 Res =
1296 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_16, DL, VT, Operand: Imm).getNode();
1297 } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
1298 Res =
1299 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_32, DL, VT, Operand: Imm).getNode();
1300 } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
1301 Res = CurDAG->getMachineNode(
1302 Opcode: Opc, dl: DL, VT, Op1: Imm,
1303 Op2: CurDAG->getTargetConstant(Val: RISCVFPRndMode::RNE, DL, VT: XLenVT));
1304 else
1305 Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: Imm);
1306
1307 // For f64 -0.0, we need to insert a fneg.d idiom.
1308 if (NegZeroF64) {
1309 Opc = RISCV::FSGNJN_D;
1310 if (HasZdinx)
1311 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
1312 Res =
1313 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: SDValue(Res, 0), Op2: SDValue(Res, 0));
1314 }
1315
1316 ReplaceNode(F: Node, T: Res);
1317 return;
1318 }
1319 case RISCVISD::BuildGPRPair:
1320 case RISCVISD::BuildPairF64:
1321 case RISCVISD::BuildPairGPRVec: {
1322 if (Opcode == RISCVISD::BuildPairF64 && !Subtarget->hasStdExtZdinx())
1323 break;
1324
1325 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::BuildPairF64) &&
1326 "BuildPairF64 only handled here on rv32i_zdinx");
1327
1328 SDValue N =
1329 buildGPRPair(CurDAG, DL, VT, Lo: Node->getOperand(Num: 0), Hi: Node->getOperand(Num: 1));
1330 ReplaceNode(F: Node, T: N.getNode());
1331 return;
1332 }
1333 case RISCVISD::SplitGPRPair:
1334 case RISCVISD::SplitF64:
1335 case RISCVISD::SplitGPRVec: {
1336 if (Subtarget->hasStdExtZdinx() || Opcode != RISCVISD::SplitF64) {
1337 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::SplitF64) &&
1338 "SplitF64 only handled here on rv32i_zdinx");
1339
1340 if (!SDValue(Node, 0).use_empty()) {
1341 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL,
1342 VT: Node->getValueType(ResNo: 0),
1343 Operand: Node->getOperand(Num: 0));
1344 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1345 }
1346
1347 if (!SDValue(Node, 1).use_empty()) {
1348 SDValue Hi = CurDAG->getTargetExtractSubreg(
1349 SRIdx: RISCV::sub_gpr_odd, DL, VT: Node->getValueType(ResNo: 1), Operand: Node->getOperand(Num: 0));
1350 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1351 }
1352
1353 CurDAG->RemoveDeadNode(N: Node);
1354 return;
1355 }
1356
1357 if (!Subtarget->hasStdExtZfa())
1358 break;
1359 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
1360 "Unexpected subtarget");
1361
1362 // With Zfa, lower to fmv.x.w and fmvh.x.d.
1363 if (!SDValue(Node, 0).use_empty()) {
1364 SDNode *Lo = CurDAG->getMachineNode(Opcode: RISCV::FMV_X_W_FPR64, dl: DL, VT,
1365 Op1: Node->getOperand(Num: 0));
1366 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Lo, 0));
1367 }
1368 if (!SDValue(Node, 1).use_empty()) {
1369 SDNode *Hi = CurDAG->getMachineNode(Opcode: RISCV::FMVH_X_D, dl: DL, VT,
1370 Op1: Node->getOperand(Num: 0));
1371 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Hi, 0));
1372 }
1373
1374 CurDAG->RemoveDeadNode(N: Node);
1375 return;
1376 }
1377 case ISD::SHL: {
1378 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1379 if (!N1C)
1380 break;
1381 SDValue N0 = Node->getOperand(Num: 0);
1382 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
1383 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1384 break;
1385 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1386 unsigned ShAmt = N1C->getZExtValue();
1387 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1388 "Unexpected shift amount!");
1389 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1390
1391 if (isShiftedMask_64(Value: Mask)) {
1392 unsigned XLen = Subtarget->getXLen();
1393 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1394 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1395 if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
1396 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
1397 // where C2 has 32 leading zeros and C3 trailing zeros.
1398 SDNode *SRLIW = CurDAG->getMachineNode(
1399 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1400 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1401 SDNode *SLLI = CurDAG->getMachineNode(
1402 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1403 Op2: CurDAG->getTargetConstant(Val: TrailingZeros + ShAmt, DL, VT));
1404 ReplaceNode(F: Node, T: SLLI);
1405 return;
1406 }
1407 if (TrailingZeros == 0 && LeadingZeros > ShAmt &&
1408 XLen - LeadingZeros > 11 && LeadingZeros != 32) {
1409 // Optimize (shl (and X, C2), C) -> (srli (slli X, C4), C4-C)
1410 // where C2 has C4 leading zeros and no trailing zeros.
1411 // This is profitable if the "and" was to be lowered to
1412 // (srli (slli X, C4), C4) and not (andi X, C2).
1413 // For "LeadingZeros == 32":
1414 // - with Zba it's just (slli.uw X, C)
1415 // - without Zba a tablegen pattern applies the very same
1416 // transform as we would have done here
1417 SDNode *SLLI = CurDAG->getMachineNode(
1418 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1419 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1420 SDNode *SRLI = CurDAG->getMachineNode(
1421 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1422 Op2: CurDAG->getTargetConstant(Val: LeadingZeros - ShAmt, DL, VT));
1423 ReplaceNode(F: Node, T: SRLI);
1424 return;
1425 }
1426 }
1427 break;
1428 }
1429 case ISD::SRL: {
1430 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1431 if (!N1C)
1432 break;
1433 SDValue N0 = Node->getOperand(Num: 0);
1434 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1435 break;
1436 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1437 unsigned ShAmt = N1C->getZExtValue();
1438 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1439 "Unexpected shift amount!");
1440 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1441
1442 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1443 // 32 leading zeros and C3 trailing zeros.
1444 if (isShiftedMask_64(Value: Mask) && N0.hasOneUse()) {
1445 unsigned XLen = Subtarget->getXLen();
1446 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1447 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1448 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1449 SDNode *SRLIW = CurDAG->getMachineNode(
1450 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1451 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1452 SDNode *SLLI = CurDAG->getMachineNode(
1453 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1454 Op2: CurDAG->getTargetConstant(Val: TrailingZeros - ShAmt, DL, VT));
1455 ReplaceNode(F: Node, T: SLLI);
1456 return;
1457 }
1458 }
1459
1460 // Optimize (srl (and X, C2), C) ->
1461 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1462 // Where C2 is a mask with C3 trailing ones.
1463 // Taking into account that the C2 may have had lower bits unset by
1464 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1465 // This pattern occurs when type legalizing right shifts for types with
1466 // less than XLen bits.
1467 Mask |= maskTrailingOnes<uint64_t>(N: ShAmt);
1468 if (!isMask_64(Value: Mask))
1469 break;
1470 unsigned TrailingOnes = llvm::countr_one(Value: Mask);
1471 if (ShAmt >= TrailingOnes)
1472 break;
1473 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1474 if (TrailingOnes == 32) {
1475 SDNode *SRLI = CurDAG->getMachineNode(
1476 Opcode: Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, dl: DL, VT,
1477 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1478 ReplaceNode(F: Node, T: SRLI);
1479 return;
1480 }
1481
1482 // Only do the remaining transforms if the AND has one use.
1483 if (!N0.hasOneUse())
1484 break;
1485
1486 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1487 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1488 SDNode *BEXTI = CurDAG->getMachineNode(
1489 Opcode: Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, dl: DL, VT,
1490 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1491 ReplaceNode(F: Node, T: BEXTI);
1492 return;
1493 }
1494
1495 const unsigned Msb = TrailingOnes - 1;
1496 const unsigned Lsb = ShAmt;
1497 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0.getOperand(i: 0), Msb, Lsb))
1498 return;
1499
1500 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1501 SDNode *SLLI =
1502 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1503 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1504 SDNode *SRLI = CurDAG->getMachineNode(
1505 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1506 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1507 ReplaceNode(F: Node, T: SRLI);
1508 return;
1509 }
1510 case ISD::SRA: {
1511 if (trySignedBitfieldExtract(Node))
1512 return;
1513
1514 if (trySignedBitfieldInsertInSign(Node))
1515 return;
1516
1517 // Optimize (sra (sext_inreg X, i16), C) ->
1518 // (srai (slli X, (XLen-16), (XLen-16) + C)
1519 // And (sra (sext_inreg X, i8), C) ->
1520 // (srai (slli X, (XLen-8), (XLen-8) + C)
1521 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1522 // This transform matches the code we get without Zbb. The shifts are more
1523 // compressible, and this can help expose CSE opportunities in the sdiv by
1524 // constant optimization.
1525 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1526 if (!N1C)
1527 break;
1528 SDValue N0 = Node->getOperand(Num: 0);
1529 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1530 break;
1531 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1532 unsigned ShAmt = N1C->getZExtValue();
1533 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1534 "Unexpected shift amount!");
1535 unsigned ExtSize =
1536 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
1537 // ExtSize of 32 should use sraiw via tablegen pattern.
1538 if (ExtSize >= 32 || ShAmt >= ExtSize)
1539 break;
1540 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1541 SDNode *SLLI =
1542 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1543 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1544 SDNode *SRAI = CurDAG->getMachineNode(
1545 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1546 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1547 ReplaceNode(F: Node, T: SRAI);
1548 return;
1549 }
1550 case ISD::SIGN_EXTEND_INREG: {
1551 // Optimize (sext_inreg (srl X, C), i8/i16) ->
1552 // (srai (slli X, XLen-ExtSize-C), XLen-ExtSize)
1553 // This is a bitfield extract pattern where we're extracting a signed
1554 // 8-bit or 16-bit field from position C.
1555 SDValue N0 = Node->getOperand(Num: 0);
1556 if (N0.getOpcode() != ISD::SRL || !N0.hasOneUse())
1557 break;
1558
1559 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1560 if (!ShAmtC)
1561 break;
1562
1563 unsigned ExtSize =
1564 cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT().getSizeInBits();
1565 unsigned ShAmt = ShAmtC->getZExtValue();
1566 unsigned XLen = Subtarget->getXLen();
1567
1568 // Only handle types less than 32, and make sure the shift amount is valid.
1569 if (ExtSize >= 32 || ShAmt >= XLen - ExtSize)
1570 break;
1571
1572 unsigned LShAmt = XLen - ExtSize - ShAmt;
1573 SDNode *SLLI =
1574 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1575 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1576 SDNode *SRAI = CurDAG->getMachineNode(
1577 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1578 Op2: CurDAG->getTargetConstant(Val: XLen - ExtSize, DL, VT));
1579 ReplaceNode(F: Node, T: SRAI);
1580 return;
1581 }
1582 case ISD::OR: {
1583 if (tryShrinkShlLogicImm(Node))
1584 return;
1585
1586 break;
1587 }
1588 case ISD::XOR:
1589 if (tryShrinkShlLogicImm(Node))
1590 return;
1591
1592 break;
1593 case ISD::AND: {
1594 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1595 if (!N1C)
1596 break;
1597 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1598
1599 SDValue N0 = Node->getOperand(Num: 0);
1600
1601 bool LeftShift = N0.getOpcode() == ISD::SHL;
1602 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1603 auto *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1604 if (!C)
1605 break;
1606 unsigned C2 = C->getZExtValue();
1607 unsigned XLen = Subtarget->getXLen();
1608 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1609
1610 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1611 // shift pair might offer more compression opportunities.
1612 // TODO: We could check for C extension here, but we don't have many lit
1613 // tests with the C extension enabled so not checking gets better
1614 // coverage.
1615 // TODO: What if ANDI faster than shift?
1616 bool IsCANDI = isInt<6>(x: N1C->getSExtValue());
1617
1618 uint64_t C1 = N1C->getZExtValue();
1619
1620 // Clear irrelevant bits in the mask.
1621 if (LeftShift)
1622 C1 &= maskTrailingZeros<uint64_t>(N: C2);
1623 else
1624 C1 &= maskTrailingOnes<uint64_t>(N: XLen - C2);
1625
1626 // Some transforms should only be done if the shift has a single use or
1627 // the AND would become (srli (slli X, 32), 32)
1628 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1629
1630 SDValue X = N0.getOperand(i: 0);
1631
1632 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1633 // with c3 leading zeros.
1634 if (!LeftShift && isMask_64(Value: C1)) {
1635 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1636 if (C2 < Leading) {
1637 // If the number of leading zeros is C2+32 this can be SRLIW.
1638 if (C2 + 32 == Leading) {
1639 SDNode *SRLIW = CurDAG->getMachineNode(
1640 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X, Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1641 ReplaceNode(F: Node, T: SRLIW);
1642 return;
1643 }
1644
1645 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1646 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1647 //
1648 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1649 // legalized and goes through DAG combine.
1650 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1651 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1652 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32) {
1653 SDNode *SRAIW =
1654 CurDAG->getMachineNode(Opcode: RISCV::SRAIW, dl: DL, VT, Op1: X.getOperand(i: 0),
1655 Op2: CurDAG->getTargetConstant(Val: 31, DL, VT));
1656 SDNode *SRLIW = CurDAG->getMachineNode(
1657 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: SDValue(SRAIW, 0),
1658 Op2: CurDAG->getTargetConstant(Val: Leading - 32, DL, VT));
1659 ReplaceNode(F: Node, T: SRLIW);
1660 return;
1661 }
1662
1663 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1664 // available.
1665 // Transform (and (srl x, C2), C1)
1666 // -> (<bfextract> x, msb, lsb)
1667 //
1668 // Make sure to keep this below the SRLIW cases, as we always want to
1669 // prefer the more common instruction.
1670 const unsigned Msb = llvm::bit_width(Value: C1) + C2 - 1;
1671 const unsigned Lsb = C2;
1672 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1673 return;
1674
1675 // (srli (slli x, c3-c2), c3).
1676 // Skip if we could use (zext.w (sraiw X, C2)).
1677 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1678 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1679 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32;
1680 // Also Skip if we can use bexti or th.tst.
1681 Skip |= HasBitTest && Leading == XLen - 1;
1682 if (OneUseOrZExtW && !Skip) {
1683 SDNode *SLLI = CurDAG->getMachineNode(
1684 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1685 Op2: CurDAG->getTargetConstant(Val: Leading - C2, DL, VT));
1686 SDNode *SRLI = CurDAG->getMachineNode(
1687 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1688 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1689 ReplaceNode(F: Node, T: SRLI);
1690 return;
1691 }
1692 }
1693 }
1694
1695 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1696 // shifted by c2 bits with c3 leading zeros.
1697 if (LeftShift && isShiftedMask_64(Value: C1)) {
1698 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1699
1700 if (C2 + Leading < XLen &&
1701 C1 == (maskTrailingOnes<uint64_t>(N: XLen - (C2 + Leading)) << C2)) {
1702 // Use slli.uw when possible.
1703 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1704 SDNode *SLLI_UW =
1705 CurDAG->getMachineNode(Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: X,
1706 Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1707 ReplaceNode(F: Node, T: SLLI_UW);
1708 return;
1709 }
1710
1711 // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
1712 // available.
1713 // Transform (and (shl x, c2), c1)
1714 // -> (<bfinsert> x, msb, lsb)
1715 // e.g.
1716 // (and (shl x, 12), 0x00fff000)
1717 // If XLen = 32 and C2 = 12, then
1718 // Msb = 32 - 8 - 1 = 23 and Lsb = 12
1719 const unsigned Msb = XLen - Leading - 1;
1720 const unsigned Lsb = C2;
1721 if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
1722 return;
1723
1724 if (OneUseOrZExtW && !IsCANDI) {
1725 // (packh x0, X)
1726 if (Subtarget->hasStdExtZbkb() && C1 == 0xff00 && C2 == 8) {
1727 SDNode *PACKH = CurDAG->getMachineNode(
1728 Opcode: RISCV::PACKH, dl: DL, VT,
1729 Op1: CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT()), Op2: X);
1730 ReplaceNode(F: Node, T: PACKH);
1731 return;
1732 }
1733 // (srli (slli c2+c3), c3)
1734 SDNode *SLLI = CurDAG->getMachineNode(
1735 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1736 Op2: CurDAG->getTargetConstant(Val: C2 + Leading, DL, VT));
1737 SDNode *SRLI = CurDAG->getMachineNode(
1738 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1739 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1740 ReplaceNode(F: Node, T: SRLI);
1741 return;
1742 }
1743 }
1744 }
1745
1746 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1747 // shifted mask with c2 leading zeros and c3 trailing zeros.
1748 if (!LeftShift && isShiftedMask_64(Value: C1)) {
1749 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1750 unsigned Trailing = llvm::countr_zero(Val: C1);
1751 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1752 !IsCANDI) {
1753 unsigned SrliOpc = RISCV::SRLI;
1754 // If the input is zexti32 we should use SRLIW.
1755 if (X.getOpcode() == ISD::AND &&
1756 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1757 X.getConstantOperandVal(i: 1) == UINT64_C(0xFFFFFFFF)) {
1758 SrliOpc = RISCV::SRLIW;
1759 X = X.getOperand(i: 0);
1760 }
1761 SDNode *SRLI = CurDAG->getMachineNode(
1762 Opcode: SrliOpc, dl: DL, VT, Op1: X,
1763 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1764 SDNode *SLLI = CurDAG->getMachineNode(
1765 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1766 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1767 ReplaceNode(F: Node, T: SLLI);
1768 return;
1769 }
1770 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1771 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1772 OneUseOrZExtW && !IsCANDI) {
1773 SDNode *SRLIW = CurDAG->getMachineNode(
1774 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1775 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1776 SDNode *SLLI = CurDAG->getMachineNode(
1777 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1778 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1779 ReplaceNode(F: Node, T: SLLI);
1780 return;
1781 }
1782 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1783 if (Trailing > 0 && Leading + Trailing == 32 && C2 + Trailing < XLen &&
1784 OneUseOrZExtW && Subtarget->hasStdExtZba()) {
1785 SDNode *SRLI = CurDAG->getMachineNode(
1786 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1787 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1788 SDNode *SLLI_UW = CurDAG->getMachineNode(
1789 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1790 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1791 ReplaceNode(F: Node, T: SLLI_UW);
1792 return;
1793 }
1794 }
1795
1796 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1797 // shifted mask with no leading zeros and c3 trailing zeros.
1798 if (LeftShift && isShiftedMask_64(Value: C1)) {
1799 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1800 unsigned Trailing = llvm::countr_zero(Val: C1);
1801 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1802 SDNode *SRLI = CurDAG->getMachineNode(
1803 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1804 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1805 SDNode *SLLI = CurDAG->getMachineNode(
1806 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1807 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1808 ReplaceNode(F: Node, T: SLLI);
1809 return;
1810 }
1811 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1812 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1813 SDNode *SRLIW = CurDAG->getMachineNode(
1814 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1815 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1816 SDNode *SLLI = CurDAG->getMachineNode(
1817 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1818 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1819 ReplaceNode(F: Node, T: SLLI);
1820 return;
1821 }
1822
1823 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1824 if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
1825 Subtarget->hasStdExtZba()) {
1826 SDNode *SRLI = CurDAG->getMachineNode(
1827 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1828 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1829 SDNode *SLLI_UW = CurDAG->getMachineNode(
1830 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1831 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1832 ReplaceNode(F: Node, T: SLLI_UW);
1833 return;
1834 }
1835 }
1836 }
1837
1838 const uint64_t C1 = N1C->getZExtValue();
1839
1840 if (N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
1841 N0.hasOneUse()) {
1842 unsigned C2 = N0.getConstantOperandVal(i: 1);
1843 unsigned XLen = Subtarget->getXLen();
1844 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1845
1846 SDValue X = N0.getOperand(i: 0);
1847
1848 // Prefer SRAIW + ANDI when possible.
1849 bool Skip = C2 > 32 && isInt<12>(x: N1C->getSExtValue()) &&
1850 X.getOpcode() == ISD::SHL &&
1851 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1852 X.getConstantOperandVal(i: 1) == 32;
1853 // Turn (and (sra x, c2), c1) -> (srli (srai x, c2-c3), c3) if c1 is a
1854 // mask with c3 leading zeros and c2 is larger than c3.
1855 if (isMask_64(Value: C1) && !Skip) {
1856 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1857 if (C2 > Leading) {
1858 SDNode *SRAI = CurDAG->getMachineNode(
1859 Opcode: RISCV::SRAI, dl: DL, VT, Op1: X,
1860 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1861 SDNode *SRLI = CurDAG->getMachineNode(
1862 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1863 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1864 ReplaceNode(F: Node, T: SRLI);
1865 return;
1866 }
1867 }
1868
1869 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
1870 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
1871 // use (slli (srli (srai y, c2 - c3), c3 + c4), c4).
1872 if (isShiftedMask_64(Value: C1) && !Skip) {
1873 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1874 unsigned Trailing = llvm::countr_zero(Val: C1);
1875 if (C2 > Leading && Leading > 0 && Trailing > 0) {
1876 SDNode *SRAI = CurDAG->getMachineNode(
1877 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1878 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1879 SDNode *SRLI = CurDAG->getMachineNode(
1880 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1881 Op2: CurDAG->getTargetConstant(Val: Leading + Trailing, DL, VT));
1882 SDNode *SLLI = CurDAG->getMachineNode(
1883 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1884 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1885 ReplaceNode(F: Node, T: SLLI);
1886 return;
1887 }
1888 }
1889 }
1890
1891 // If C1 masks off the upper bits only (but can't be formed as an
1892 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1893 // available.
1894 // Transform (and x, C1)
1895 // -> (<bfextract> x, msb, lsb)
1896 if (isMask_64(Value: C1) && !isInt<12>(x: N1C->getSExtValue()) &&
1897 !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
1898 !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
1899 const unsigned Msb = llvm::bit_width(Value: C1) - 1;
1900 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0, Msb, Lsb: 0))
1901 return;
1902 }
1903
1904 if (tryShrinkShlLogicImm(Node))
1905 return;
1906
1907 break;
1908 }
1909 case ISD::MUL: {
1910 // Special case for calculating (mul (and X, C2), C1) where the full product
1911 // fits in XLen bits. We can shift X left by the number of leading zeros in
1912 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1913 // product has XLen trailing zeros, putting it in the output of MULHU. This
1914 // can avoid materializing a constant in a register for C2.
1915
1916 // RHS should be a constant.
1917 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1918 if (!N1C || !N1C->hasOneUse())
1919 break;
1920
1921 // LHS should be an AND with constant.
1922 SDValue N0 = Node->getOperand(Num: 0);
1923 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1924 break;
1925
1926 uint64_t C2 = N0.getConstantOperandVal(i: 1);
1927
1928 // Constant should be a mask.
1929 if (!isMask_64(Value: C2))
1930 break;
1931
1932 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1933 // multiple users or the constant is a simm12. This prevents inserting a
1934 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1935 // make it more costly to materialize. Otherwise, using a SLLI might allow
1936 // it to be compressed.
1937 bool IsANDIOrZExt =
1938 isInt<12>(x: C2) ||
1939 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1940 // With XTHeadBb, we can use TH.EXTU.
1941 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1942 if (IsANDIOrZExt && (isInt<12>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1943 break;
1944 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1945 // the constant is a simm32.
1946 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1947 // With XTHeadBb, we can use TH.EXTU.
1948 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1949 if (IsZExtW && (isInt<32>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1950 break;
1951
1952 // We need to shift left the AND input and C1 by a total of XLen bits.
1953
1954 // How far left do we need to shift the AND input?
1955 unsigned XLen = Subtarget->getXLen();
1956 unsigned LeadingZeros = XLen - llvm::bit_width(Value: C2);
1957
1958 // The constant gets shifted by the remaining amount unless that would
1959 // shift bits out.
1960 uint64_t C1 = N1C->getZExtValue();
1961 unsigned ConstantShift = XLen - LeadingZeros;
1962 if (ConstantShift > (XLen - llvm::bit_width(Value: C1)))
1963 break;
1964
1965 uint64_t ShiftedC1 = C1 << ConstantShift;
1966 // If this RV32, we need to sign extend the constant.
1967 if (XLen == 32)
1968 ShiftedC1 = SignExtend64<32>(x: ShiftedC1);
1969
1970 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1971 SDNode *Imm = selectImm(CurDAG, DL, VT, Imm: ShiftedC1, Subtarget: *Subtarget).getNode();
1972 SDNode *SLLI =
1973 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1974 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1975 SDNode *MULHU = CurDAG->getMachineNode(Opcode: RISCV::MULHU, dl: DL, VT,
1976 Op1: SDValue(SLLI, 0), Op2: SDValue(Imm, 0));
1977 ReplaceNode(F: Node, T: MULHU);
1978 return;
1979 }
1980 case ISD::SMUL_LOHI:
1981 case ISD::UMUL_LOHI:
1982 case RISCVISD::WMULSU:
1983 case RISCVISD::WADD:
1984 case RISCVISD::WSUB:
1985 case RISCVISD::WADDU:
1986 case RISCVISD::WSUBU: {
1987 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1988 "Unexpected opcode");
1989
1990 unsigned Opc;
1991 switch (Node->getOpcode()) {
1992 default:
1993 llvm_unreachable("Unexpected opcode");
1994 case ISD::SMUL_LOHI:
1995 Opc = RISCV::WMUL;
1996 break;
1997 case ISD::UMUL_LOHI:
1998 Opc = RISCV::WMULU;
1999 break;
2000 case RISCVISD::WMULSU:
2001 Opc = RISCV::WMULSU;
2002 break;
2003 case RISCVISD::WADD:
2004 Opc = RISCV::WADD;
2005 break;
2006 case RISCVISD::WSUB:
2007 Opc = RISCV::WSUB;
2008 break;
2009 case RISCVISD::WADDU:
2010 Opc = RISCV::WADDU;
2011 break;
2012 case RISCVISD::WSUBU:
2013 Opc = RISCV::WSUBU;
2014 break;
2015 }
2016
2017 SDNode *Result = CurDAG->getMachineNode(
2018 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Node->getOperand(Num: 0), Op2: Node->getOperand(Num: 1));
2019
2020 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(Result, 0));
2021 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2022 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2023 CurDAG->RemoveDeadNode(N: Node);
2024 return;
2025 }
2026 case RISCVISD::WSLL:
2027 case RISCVISD::WSLA: {
2028 // Custom select WSLL/WSLA for RV32P.
2029 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
2030 "Unexpected opcode");
2031
2032 bool IsSigned = Node->getOpcode() == RISCVISD::WSLA;
2033
2034 SDValue ShAmt = Node->getOperand(Num: 1);
2035
2036 unsigned Opc;
2037
2038 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShAmt);
2039 if (ShAmtC && ShAmtC->getZExtValue() < 64) {
2040 Opc = IsSigned ? RISCV::WSLAI : RISCV::WSLLI;
2041 ShAmt = CurDAG->getTargetConstant(Val: ShAmtC->getZExtValue(), DL, VT: XLenVT);
2042 } else {
2043 Opc = IsSigned ? RISCV::WSLA : RISCV::WSLL;
2044 }
2045
2046 SDNode *WShift = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped,
2047 Op1: Node->getOperand(Num: 0), Op2: ShAmt);
2048
2049 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(WShift, 0));
2050 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2051 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2052 CurDAG->RemoveDeadNode(N: Node);
2053 return;
2054 }
2055 case ISD::LOAD: {
2056 if (tryIndexedLoad(Node))
2057 return;
2058
2059 if (Subtarget->hasVendorXCVmem() && !Subtarget->is64Bit()) {
2060 // We match post-incrementing load here
2061 LoadSDNode *Load = cast<LoadSDNode>(Val: Node);
2062 if (Load->getAddressingMode() != ISD::POST_INC)
2063 break;
2064
2065 SDValue Chain = Node->getOperand(Num: 0);
2066 SDValue Base = Node->getOperand(Num: 1);
2067 SDValue Offset = Node->getOperand(Num: 2);
2068
2069 bool Simm12 = false;
2070 bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
2071
2072 if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Val&: Offset)) {
2073 int ConstantVal = ConstantOffset->getSExtValue();
2074 Simm12 = isInt<12>(x: ConstantVal);
2075 if (Simm12)
2076 Offset = CurDAG->getSignedTargetConstant(Val: ConstantVal, DL: SDLoc(Offset),
2077 VT: Offset.getValueType());
2078 }
2079
2080 unsigned Opcode = 0;
2081 switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
2082 case MVT::i8:
2083 if (Simm12 && SignExtend)
2084 Opcode = RISCV::CV_LB_ri_inc;
2085 else if (Simm12 && !SignExtend)
2086 Opcode = RISCV::CV_LBU_ri_inc;
2087 else if (!Simm12 && SignExtend)
2088 Opcode = RISCV::CV_LB_rr_inc;
2089 else
2090 Opcode = RISCV::CV_LBU_rr_inc;
2091 break;
2092 case MVT::i16:
2093 if (Simm12 && SignExtend)
2094 Opcode = RISCV::CV_LH_ri_inc;
2095 else if (Simm12 && !SignExtend)
2096 Opcode = RISCV::CV_LHU_ri_inc;
2097 else if (!Simm12 && SignExtend)
2098 Opcode = RISCV::CV_LH_rr_inc;
2099 else
2100 Opcode = RISCV::CV_LHU_rr_inc;
2101 break;
2102 case MVT::i32:
2103 if (Simm12)
2104 Opcode = RISCV::CV_LW_ri_inc;
2105 else
2106 Opcode = RISCV::CV_LW_rr_inc;
2107 break;
2108 default:
2109 break;
2110 }
2111 if (!Opcode)
2112 break;
2113
2114 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode, dl: DL, VT1: XLenVT, VT2: XLenVT,
2115 VT3: Chain.getSimpleValueType(), Op1: Base,
2116 Op2: Offset, Op3: Chain));
2117 return;
2118 }
2119 break;
2120 }
2121 case RISCVISD::LD_RV32: {
2122 assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd");
2123
2124 SDValue Base, Offset;
2125 SDValue Chain = Node->getOperand(Num: 0);
2126 SDValue Addr = Node->getOperand(Num: 1);
2127 SelectAddrRegImm(Addr, Base, Offset);
2128
2129 SDValue Ops[] = {Base, Offset, Chain};
2130 MachineSDNode *New = CurDAG->getMachineNode(
2131 Opcode: RISCV::LD_RV32, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2132 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2133 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2134 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2135 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2136 ReplaceUses(F: SDValue(Node, 2), T: SDValue(New, 1));
2137 CurDAG->RemoveDeadNode(N: Node);
2138 return;
2139 }
2140 case RISCVISD::SD_RV32: {
2141 SDValue Base, Offset;
2142 SDValue Chain = Node->getOperand(Num: 0);
2143 SDValue Addr = Node->getOperand(Num: 3);
2144 SelectAddrRegImm(Addr, Base, Offset);
2145
2146 SDValue Lo = Node->getOperand(Num: 1);
2147 SDValue Hi = Node->getOperand(Num: 2);
2148
2149 SDValue RegPair;
2150 // Peephole to use X0_Pair for storing zero.
2151 if (isNullConstant(V: Lo) && isNullConstant(V: Hi)) {
2152 RegPair = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2153 } else {
2154 RegPair = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo, Hi);
2155 }
2156
2157 MachineSDNode *New = CurDAG->getMachineNode(Opcode: RISCV::SD_RV32, dl: DL, VT: MVT::Other,
2158 Ops: {RegPair, Base, Offset, Chain});
2159 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2160 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0));
2161 CurDAG->RemoveDeadNode(N: Node);
2162 return;
2163 }
2164 case RISCVISD::MQWACC:
2165 case RISCVISD::MQRWACC:
2166 case RISCVISD::WMACC:
2167 case RISCVISD::WMACCU:
2168 case RISCVISD::WMACCSU:
2169 case RISCVISD::PM2WADDA:
2170 case RISCVISD::PM2WADDA_X:
2171 case RISCVISD::PM2WADDAU:
2172 case RISCVISD::PM2WSUBA:
2173 case RISCVISD::PM2WSUBA_X:
2174 case RISCVISD::PM2WADDASU:
2175 case RISCVISD::WADDAU:
2176 case RISCVISD::WSUBAU:
2177 case RISCVISD::WADDA:
2178 case RISCVISD::WSUBA: {
2179 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2180 "Unexpected opcode");
2181
2182 SDValue Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Node->getOperand(Num: 0),
2183 Hi: Node->getOperand(Num: 1));
2184 unsigned Opc;
2185 switch (Opcode) {
2186 default:
2187 llvm_unreachable("Unexpected opcode");
2188 case RISCVISD::MQWACC:
2189 Opc = RISCV::MQWACC;
2190 break;
2191 case RISCVISD::MQRWACC:
2192 Opc = RISCV::MQRWACC;
2193 break;
2194 case RISCVISD::WMACC:
2195 Opc = RISCV::WMACC;
2196 break;
2197 case RISCVISD::WMACCU:
2198 Opc = RISCV::WMACCU;
2199 break;
2200 case RISCVISD::WMACCSU:
2201 Opc = RISCV::WMACCSU;
2202 break;
2203 case RISCVISD::PM2WADDA:
2204 Opc = RISCV::PM2WADDA_H;
2205 break;
2206 case RISCVISD::PM2WADDA_X:
2207 Opc = RISCV::PM2WADDA_HX;
2208 break;
2209 case RISCVISD::PM2WADDAU:
2210 Opc = RISCV::PM2WADDAU_H;
2211 break;
2212 case RISCVISD::PM2WSUBA:
2213 Opc = RISCV::PM2WSUBA_H;
2214 break;
2215 case RISCVISD::PM2WSUBA_X:
2216 Opc = RISCV::PM2WSUBA_HX;
2217 break;
2218 case RISCVISD::PM2WADDASU:
2219 Opc = RISCV::PM2WADDASU_H;
2220 break;
2221 case RISCVISD::WADDAU:
2222 Opc = RISCV::WADDAU;
2223 break;
2224 case RISCVISD::WSUBAU:
2225 Opc = RISCV::WSUBAU;
2226 break;
2227 case RISCVISD::WADDA:
2228 Opc = RISCV::WADDA;
2229 break;
2230 case RISCVISD::WSUBA:
2231 Opc = RISCV::WSUBA;
2232 break;
2233 }
2234 MachineSDNode *New = CurDAG->getMachineNode(
2235 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Node->getOperand(Num: 2), Op3: Node->getOperand(Num: 3));
2236 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2237 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2238 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2239 CurDAG->RemoveDeadNode(N: Node);
2240 return;
2241 }
2242 case RISCVISD::PM2WADD:
2243 case RISCVISD::PM2WADDU:
2244 case RISCVISD::PM2WADDSU:
2245 case RISCVISD::PM2WADD_X:
2246 case RISCVISD::PM2WSUB:
2247 case RISCVISD::PM2WSUB_X: {
2248 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2249 "Unexpected opcode");
2250 unsigned Opc;
2251 switch (Opcode) {
2252 default:
2253 llvm_unreachable("Unexpected opcode");
2254 case RISCVISD::PM2WADD:
2255 Opc = RISCV::PM2WADD_H;
2256 break;
2257 case RISCVISD::PM2WADDU:
2258 Opc = RISCV::PM2WADDU_H;
2259 break;
2260 case RISCVISD::PM2WADDSU:
2261 Opc = RISCV::PM2WADDSU_H;
2262 break;
2263 case RISCVISD::PM2WADD_X:
2264 Opc = RISCV::PM2WADD_HX;
2265 break;
2266 case RISCVISD::PM2WSUB:
2267 Opc = RISCV::PM2WSUB_H;
2268 break;
2269 case RISCVISD::PM2WSUB_X:
2270 Opc = RISCV::PM2WSUB_HX;
2271 break;
2272 }
2273 MachineSDNode *New = CurDAG->getMachineNode(
2274 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Node->getOperand(Num: 0), Op2: Node->getOperand(Num: 1));
2275 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2276 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2277 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2278 CurDAG->RemoveDeadNode(N: Node);
2279 return;
2280 }
2281 case RISCVISD::ADDD:
2282 // Try to match WMACC pattern: ADDD where one operand pair comes from a
2283 // widening multiply.
2284 if (tryWideningMulAcc(Node, DL))
2285 return;
2286
2287 // Fall through to regular ADDD selection.
2288 [[fallthrough]];
2289 case RISCVISD::SUBD: {
2290 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2291 "Unexpected opcode");
2292
2293 SDValue Op0Lo = Node->getOperand(Num: 0);
2294 SDValue Op0Hi = Node->getOperand(Num: 1);
2295
2296 SDValue Op0;
2297 if (isNullConstant(V: Op0Lo) && isNullConstant(V: Op0Hi)) {
2298 Op0 = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2299 } else {
2300 Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op0Lo, Hi: Op0Hi);
2301 }
2302
2303 SDValue Op1Lo = Node->getOperand(Num: 2);
2304 SDValue Op1Hi = Node->getOperand(Num: 3);
2305 SDValue Op1 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op1Lo, Hi: Op1Hi);
2306
2307 unsigned Opc = Opcode == RISCVISD::ADDD ? RISCV::ADDD : RISCV::SUBD;
2308 MachineSDNode *New =
2309 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1);
2310
2311 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2312 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2313 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2314 CurDAG->RemoveDeadNode(N: Node);
2315 return;
2316 }
2317 case ISD::INTRINSIC_WO_CHAIN: {
2318 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
2319 switch (IntNo) {
2320 // By default we do not custom select any intrinsic.
2321 default:
2322 break;
2323 case Intrinsic::riscv_vmsgeu:
2324 case Intrinsic::riscv_vmsge: {
2325 SDValue Src1 = Node->getOperand(Num: 1);
2326 SDValue Src2 = Node->getOperand(Num: 2);
2327 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
2328 bool IsCmpConstant = false;
2329 bool IsCmpMinimum = false;
2330 // Only custom select scalar second operand.
2331 if (Src2.getValueType() != XLenVT)
2332 break;
2333 // Small constants are handled with patterns.
2334 int64_t CVal = 0;
2335 MVT Src1VT = Src1.getSimpleValueType();
2336 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2337 IsCmpConstant = true;
2338 CVal = C->getSExtValue();
2339 if (CVal >= -15 && CVal <= 16) {
2340 if (!IsUnsigned || CVal != 0)
2341 break;
2342 IsCmpMinimum = true;
2343 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2344 numBits: Src1VT.getScalarSizeInBits())
2345 .getSExtValue()) {
2346 IsCmpMinimum = true;
2347 }
2348 }
2349 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode, VMSGTOpcode;
2350 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2351 default:
2352 llvm_unreachable("Unexpected LMUL!");
2353#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2354 case RISCVVType::lmulenum: \
2355 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2356 : RISCV::PseudoVMSLT_VX_##suffix; \
2357 VMSGTOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix \
2358 : RISCV::PseudoVMSGT_VX_##suffix; \
2359 break;
2360 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2361 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2362 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2363 CASE_VMSLT_OPCODES(LMUL_1, M1)
2364 CASE_VMSLT_OPCODES(LMUL_2, M2)
2365 CASE_VMSLT_OPCODES(LMUL_4, M4)
2366 CASE_VMSLT_OPCODES(LMUL_8, M8)
2367#undef CASE_VMSLT_OPCODES
2368 }
2369 // Mask operations use the LMUL from the mask type.
2370 switch (RISCVTargetLowering::getLMUL(VT)) {
2371 default:
2372 llvm_unreachable("Unexpected LMUL!");
2373#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix) \
2374 case RISCVVType::lmulenum: \
2375 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
2376 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix; \
2377 break;
2378 CASE_VMNAND_VMSET_OPCODES(LMUL_F8, B64)
2379 CASE_VMNAND_VMSET_OPCODES(LMUL_F4, B32)
2380 CASE_VMNAND_VMSET_OPCODES(LMUL_F2, B16)
2381 CASE_VMNAND_VMSET_OPCODES(LMUL_1, B8)
2382 CASE_VMNAND_VMSET_OPCODES(LMUL_2, B4)
2383 CASE_VMNAND_VMSET_OPCODES(LMUL_4, B2)
2384 CASE_VMNAND_VMSET_OPCODES(LMUL_8, B1)
2385#undef CASE_VMNAND_VMSET_OPCODES
2386 }
2387 SDValue SEW = CurDAG->getTargetConstant(
2388 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2389 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2390 SDValue VL;
2391 selectVLOp(N: Node->getOperand(Num: 3), VL);
2392
2393 // If vmsge(u) with minimum value, expand it to vmset.
2394 if (IsCmpMinimum) {
2395 ReplaceNode(F: Node,
2396 T: CurDAG->getMachineNode(Opcode: VMSetOpcode, dl: DL, VT, Op1: VL, Op2: MaskSEW));
2397 return;
2398 }
2399
2400 if (IsCmpConstant) {
2401 SDValue Imm =
2402 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2403
2404 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMSGTOpcode, dl: DL, VT,
2405 Ops: {Src1, Imm, VL, SEW}));
2406 return;
2407 }
2408
2409 // Expand to
2410 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
2411 SDValue Cmp = SDValue(
2412 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2413 0);
2414 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMNANDOpcode, dl: DL, VT,
2415 Ops: {Cmp, Cmp, VL, MaskSEW}));
2416 return;
2417 }
2418 case Intrinsic::riscv_vmsgeu_mask:
2419 case Intrinsic::riscv_vmsge_mask: {
2420 SDValue Src1 = Node->getOperand(Num: 2);
2421 SDValue Src2 = Node->getOperand(Num: 3);
2422 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
2423 bool IsCmpConstant = false;
2424 bool IsCmpMinimum = false;
2425 // Only custom select scalar second operand.
2426 if (Src2.getValueType() != XLenVT)
2427 break;
2428 // Small constants are handled with patterns.
2429 MVT Src1VT = Src1.getSimpleValueType();
2430 int64_t CVal = 0;
2431 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2432 IsCmpConstant = true;
2433 CVal = C->getSExtValue();
2434 if (CVal >= -15 && CVal <= 16) {
2435 if (!IsUnsigned || CVal != 0)
2436 break;
2437 IsCmpMinimum = true;
2438 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2439 numBits: Src1VT.getScalarSizeInBits())
2440 .getSExtValue()) {
2441 IsCmpMinimum = true;
2442 }
2443 }
2444 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
2445 VMOROpcode, VMSGTMaskOpcode;
2446 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2447 default:
2448 llvm_unreachable("Unexpected LMUL!");
2449#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2450 case RISCVVType::lmulenum: \
2451 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2452 : RISCV::PseudoVMSLT_VX_##suffix; \
2453 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
2454 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
2455 VMSGTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix##_MASK \
2456 : RISCV::PseudoVMSGT_VX_##suffix##_MASK; \
2457 break;
2458 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2459 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2460 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2461 CASE_VMSLT_OPCODES(LMUL_1, M1)
2462 CASE_VMSLT_OPCODES(LMUL_2, M2)
2463 CASE_VMSLT_OPCODES(LMUL_4, M4)
2464 CASE_VMSLT_OPCODES(LMUL_8, M8)
2465#undef CASE_VMSLT_OPCODES
2466 }
2467 // Mask operations use the LMUL from the mask type.
2468 switch (RISCVTargetLowering::getLMUL(VT)) {
2469 default:
2470 llvm_unreachable("Unexpected LMUL!");
2471#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
2472 case RISCVVType::lmulenum: \
2473 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
2474 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
2475 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
2476 break;
2477 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, B64)
2478 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, B32)
2479 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, B16)
2480 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, B8)
2481 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, B4)
2482 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, B2)
2483 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, B1)
2484#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
2485 }
2486 SDValue SEW = CurDAG->getTargetConstant(
2487 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2488 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2489 SDValue VL;
2490 selectVLOp(N: Node->getOperand(Num: 5), VL);
2491 SDValue MaskedOff = Node->getOperand(Num: 1);
2492 SDValue Mask = Node->getOperand(Num: 4);
2493
2494 // If vmsge(u) with minimum value, expand it to vmor mask, maskedoff.
2495 if (IsCmpMinimum) {
2496 // We don't need vmor if the MaskedOff and the Mask are the same
2497 // value.
2498 if (Mask == MaskedOff) {
2499 ReplaceUses(F: Node, T: Mask.getNode());
2500 return;
2501 }
2502 ReplaceNode(F: Node,
2503 T: CurDAG->getMachineNode(Opcode: VMOROpcode, dl: DL, VT,
2504 Ops: {Mask, MaskedOff, VL, MaskSEW}));
2505 return;
2506 }
2507
2508 // If the MaskedOff value and the Mask are the same value use
2509 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
2510 // This avoids needing to copy v0 to vd before starting the next sequence.
2511 if (Mask == MaskedOff) {
2512 SDValue Cmp = SDValue(
2513 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2514 0);
2515 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMANDNOpcode, dl: DL, VT,
2516 Ops: {Mask, Cmp, VL, MaskSEW}));
2517 return;
2518 }
2519
2520 SDValue PolicyOp =
2521 CurDAG->getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
2522
2523 if (IsCmpConstant) {
2524 SDValue Imm =
2525 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2526
2527 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
2528 Opcode: VMSGTMaskOpcode, dl: DL, VT,
2529 Ops: {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp}));
2530 return;
2531 }
2532
2533 // Otherwise use
2534 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
2535 // The result is mask undisturbed.
2536 // We use the same instructions to emulate mask agnostic behavior, because
2537 // the agnostic result can be either undisturbed or all 1.
2538 SDValue Cmp = SDValue(CurDAG->getMachineNode(Opcode: VMSLTMaskOpcode, dl: DL, VT,
2539 Ops: {MaskedOff, Src1, Src2, Mask,
2540 VL, SEW, PolicyOp}),
2541 0);
2542 // vmxor.mm vd, vd, v0 is used to update active value.
2543 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMXOROpcode, dl: DL, VT,
2544 Ops: {Cmp, Mask, VL, MaskSEW}));
2545 return;
2546 }
2547 case Intrinsic::riscv_vsetvli:
2548 case Intrinsic::riscv_vsetvlimax:
2549 return selectVSETVLI(Node);
2550 case Intrinsic::riscv_sf_vsettnt:
2551 case Intrinsic::riscv_sf_vsettm:
2552 case Intrinsic::riscv_sf_vsettk:
2553 return selectXSfmmVSET(Node);
2554 }
2555 break;
2556 }
2557 case ISD::INTRINSIC_W_CHAIN: {
2558 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2559 switch (IntNo) {
2560 // By default we do not custom select any intrinsic.
2561 default:
2562 break;
2563 case Intrinsic::riscv_vlseg2:
2564 case Intrinsic::riscv_vlseg3:
2565 case Intrinsic::riscv_vlseg4:
2566 case Intrinsic::riscv_vlseg5:
2567 case Intrinsic::riscv_vlseg6:
2568 case Intrinsic::riscv_vlseg7:
2569 case Intrinsic::riscv_vlseg8: {
2570 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2571 /*IsStrided*/ false);
2572 return;
2573 }
2574 case Intrinsic::riscv_vlseg2_mask:
2575 case Intrinsic::riscv_vlseg3_mask:
2576 case Intrinsic::riscv_vlseg4_mask:
2577 case Intrinsic::riscv_vlseg5_mask:
2578 case Intrinsic::riscv_vlseg6_mask:
2579 case Intrinsic::riscv_vlseg7_mask:
2580 case Intrinsic::riscv_vlseg8_mask: {
2581 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2582 /*IsStrided*/ false);
2583 return;
2584 }
2585 case Intrinsic::riscv_vlsseg2:
2586 case Intrinsic::riscv_vlsseg3:
2587 case Intrinsic::riscv_vlsseg4:
2588 case Intrinsic::riscv_vlsseg5:
2589 case Intrinsic::riscv_vlsseg6:
2590 case Intrinsic::riscv_vlsseg7:
2591 case Intrinsic::riscv_vlsseg8: {
2592 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2593 /*IsStrided*/ true);
2594 return;
2595 }
2596 case Intrinsic::riscv_vlsseg2_mask:
2597 case Intrinsic::riscv_vlsseg3_mask:
2598 case Intrinsic::riscv_vlsseg4_mask:
2599 case Intrinsic::riscv_vlsseg5_mask:
2600 case Intrinsic::riscv_vlsseg6_mask:
2601 case Intrinsic::riscv_vlsseg7_mask:
2602 case Intrinsic::riscv_vlsseg8_mask: {
2603 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2604 /*IsStrided*/ true);
2605 return;
2606 }
2607 case Intrinsic::riscv_vloxseg2:
2608 case Intrinsic::riscv_vloxseg3:
2609 case Intrinsic::riscv_vloxseg4:
2610 case Intrinsic::riscv_vloxseg5:
2611 case Intrinsic::riscv_vloxseg6:
2612 case Intrinsic::riscv_vloxseg7:
2613 case Intrinsic::riscv_vloxseg8:
2614 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2615 /*IsOrdered*/ true);
2616 return;
2617 case Intrinsic::riscv_vluxseg2:
2618 case Intrinsic::riscv_vluxseg3:
2619 case Intrinsic::riscv_vluxseg4:
2620 case Intrinsic::riscv_vluxseg5:
2621 case Intrinsic::riscv_vluxseg6:
2622 case Intrinsic::riscv_vluxseg7:
2623 case Intrinsic::riscv_vluxseg8:
2624 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2625 /*IsOrdered*/ false);
2626 return;
2627 case Intrinsic::riscv_vloxseg2_mask:
2628 case Intrinsic::riscv_vloxseg3_mask:
2629 case Intrinsic::riscv_vloxseg4_mask:
2630 case Intrinsic::riscv_vloxseg5_mask:
2631 case Intrinsic::riscv_vloxseg6_mask:
2632 case Intrinsic::riscv_vloxseg7_mask:
2633 case Intrinsic::riscv_vloxseg8_mask:
2634 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2635 /*IsOrdered*/ true);
2636 return;
2637 case Intrinsic::riscv_vluxseg2_mask:
2638 case Intrinsic::riscv_vluxseg3_mask:
2639 case Intrinsic::riscv_vluxseg4_mask:
2640 case Intrinsic::riscv_vluxseg5_mask:
2641 case Intrinsic::riscv_vluxseg6_mask:
2642 case Intrinsic::riscv_vluxseg7_mask:
2643 case Intrinsic::riscv_vluxseg8_mask:
2644 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2645 /*IsOrdered*/ false);
2646 return;
2647 case Intrinsic::riscv_vlseg8ff:
2648 case Intrinsic::riscv_vlseg7ff:
2649 case Intrinsic::riscv_vlseg6ff:
2650 case Intrinsic::riscv_vlseg5ff:
2651 case Intrinsic::riscv_vlseg4ff:
2652 case Intrinsic::riscv_vlseg3ff:
2653 case Intrinsic::riscv_vlseg2ff: {
2654 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false);
2655 return;
2656 }
2657 case Intrinsic::riscv_vlseg8ff_mask:
2658 case Intrinsic::riscv_vlseg7ff_mask:
2659 case Intrinsic::riscv_vlseg6ff_mask:
2660 case Intrinsic::riscv_vlseg5ff_mask:
2661 case Intrinsic::riscv_vlseg4ff_mask:
2662 case Intrinsic::riscv_vlseg3ff_mask:
2663 case Intrinsic::riscv_vlseg2ff_mask: {
2664 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true);
2665 return;
2666 }
2667 case Intrinsic::riscv_vloxei:
2668 case Intrinsic::riscv_vloxei_mask:
2669 case Intrinsic::riscv_vluxei:
2670 case Intrinsic::riscv_vluxei_mask: {
2671 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
2672 IntNo == Intrinsic::riscv_vluxei_mask;
2673 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
2674 IntNo == Intrinsic::riscv_vloxei_mask;
2675
2676 MVT VT = Node->getSimpleValueType(ResNo: 0);
2677 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2678
2679 unsigned CurOp = 2;
2680 SmallVector<SDValue, 8> Operands;
2681 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2682
2683 MVT IndexVT;
2684 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2685 /*IsStridedOrIndexed*/ true, Operands,
2686 /*IsLoad=*/true, IndexVT: &IndexVT);
2687
2688 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2689 "Element count mismatch");
2690
2691 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2692 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2693 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2694 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2695 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2696 "index values when XLEN=32");
2697 }
2698 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
2699 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
2700 IndexLMUL: static_cast<unsigned>(IndexLMUL));
2701 MachineSDNode *Load =
2702 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2703
2704 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2705
2706 ReplaceNode(F: Node, T: Load);
2707 return;
2708 }
2709 case Intrinsic::riscv_vlm:
2710 case Intrinsic::riscv_vle:
2711 case Intrinsic::riscv_vle_mask:
2712 case Intrinsic::riscv_vlse:
2713 case Intrinsic::riscv_vlse_mask: {
2714 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
2715 IntNo == Intrinsic::riscv_vlse_mask;
2716 bool IsStrided =
2717 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
2718
2719 MVT VT = Node->getSimpleValueType(ResNo: 0);
2720 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2721
2722 // The riscv_vlm intrinsic are always tail agnostic and no passthru
2723 // operand at the IR level. In pseudos, they have both policy and
2724 // passthru operand. The passthru operand is needed to track the
2725 // "tail undefined" state, and the policy is there just for
2726 // for consistency - it will always be "don't care" for the
2727 // unmasked form.
2728 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
2729 unsigned CurOp = 2;
2730 SmallVector<SDValue, 8> Operands;
2731 if (HasPassthruOperand)
2732 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2733 else {
2734 // We eagerly lower to implicit_def (instead of undef), as we
2735 // otherwise fail to select nodes such as: nxv1i1 = undef
2736 SDNode *Passthru =
2737 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
2738 Operands.push_back(Elt: SDValue(Passthru, 0));
2739 }
2740 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2741 Operands, /*IsLoad=*/true);
2742
2743 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2744 const RISCV::VLEPseudo *P =
2745 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
2746 LMUL: static_cast<unsigned>(LMUL));
2747 MachineSDNode *Load =
2748 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2749
2750 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2751
2752 ReplaceNode(F: Node, T: Load);
2753 return;
2754 }
2755 case Intrinsic::riscv_vleff:
2756 case Intrinsic::riscv_vleff_mask: {
2757 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
2758
2759 MVT VT = Node->getSimpleValueType(ResNo: 0);
2760 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2761
2762 unsigned CurOp = 2;
2763 SmallVector<SDValue, 7> Operands;
2764 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2765 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2766 /*IsStridedOrIndexed*/ false, Operands,
2767 /*IsLoad=*/true);
2768
2769 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2770 const RISCV::VLEPseudo *P =
2771 RISCV::getVLEPseudo(Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
2772 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2773 MachineSDNode *Load = CurDAG->getMachineNode(
2774 Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2775 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2776
2777 ReplaceNode(F: Node, T: Load);
2778 return;
2779 }
2780 case Intrinsic::riscv_nds_vln:
2781 case Intrinsic::riscv_nds_vln_mask:
2782 case Intrinsic::riscv_nds_vlnu:
2783 case Intrinsic::riscv_nds_vlnu_mask: {
2784 bool IsMasked = IntNo == Intrinsic::riscv_nds_vln_mask ||
2785 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2786 bool IsUnsigned = IntNo == Intrinsic::riscv_nds_vlnu ||
2787 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2788
2789 MVT VT = Node->getSimpleValueType(ResNo: 0);
2790 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2791 unsigned CurOp = 2;
2792 SmallVector<SDValue, 8> Operands;
2793
2794 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2795 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2796 /*IsStridedOrIndexed=*/false, Operands,
2797 /*IsLoad=*/true);
2798
2799 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2800 const RISCV::NDSVLNPseudo *P = RISCV::getNDSVLNPseudo(
2801 Masked: IsMasked, Unsigned: IsUnsigned, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2802 MachineSDNode *Load =
2803 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2804
2805 if (auto *MemOp = dyn_cast<MemSDNode>(Val: Node))
2806 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {MemOp->getMemOperand()});
2807
2808 ReplaceNode(F: Node, T: Load);
2809 return;
2810 }
2811 }
2812 break;
2813 }
2814 case ISD::INTRINSIC_VOID: {
2815 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2816 switch (IntNo) {
2817 case Intrinsic::riscv_vsseg2:
2818 case Intrinsic::riscv_vsseg3:
2819 case Intrinsic::riscv_vsseg4:
2820 case Intrinsic::riscv_vsseg5:
2821 case Intrinsic::riscv_vsseg6:
2822 case Intrinsic::riscv_vsseg7:
2823 case Intrinsic::riscv_vsseg8: {
2824 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2825 /*IsStrided*/ false);
2826 return;
2827 }
2828 case Intrinsic::riscv_vsseg2_mask:
2829 case Intrinsic::riscv_vsseg3_mask:
2830 case Intrinsic::riscv_vsseg4_mask:
2831 case Intrinsic::riscv_vsseg5_mask:
2832 case Intrinsic::riscv_vsseg6_mask:
2833 case Intrinsic::riscv_vsseg7_mask:
2834 case Intrinsic::riscv_vsseg8_mask: {
2835 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2836 /*IsStrided*/ false);
2837 return;
2838 }
2839 case Intrinsic::riscv_vssseg2:
2840 case Intrinsic::riscv_vssseg3:
2841 case Intrinsic::riscv_vssseg4:
2842 case Intrinsic::riscv_vssseg5:
2843 case Intrinsic::riscv_vssseg6:
2844 case Intrinsic::riscv_vssseg7:
2845 case Intrinsic::riscv_vssseg8: {
2846 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2847 /*IsStrided*/ true);
2848 return;
2849 }
2850 case Intrinsic::riscv_vssseg2_mask:
2851 case Intrinsic::riscv_vssseg3_mask:
2852 case Intrinsic::riscv_vssseg4_mask:
2853 case Intrinsic::riscv_vssseg5_mask:
2854 case Intrinsic::riscv_vssseg6_mask:
2855 case Intrinsic::riscv_vssseg7_mask:
2856 case Intrinsic::riscv_vssseg8_mask: {
2857 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2858 /*IsStrided*/ true);
2859 return;
2860 }
2861 case Intrinsic::riscv_vsoxseg2:
2862 case Intrinsic::riscv_vsoxseg3:
2863 case Intrinsic::riscv_vsoxseg4:
2864 case Intrinsic::riscv_vsoxseg5:
2865 case Intrinsic::riscv_vsoxseg6:
2866 case Intrinsic::riscv_vsoxseg7:
2867 case Intrinsic::riscv_vsoxseg8:
2868 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2869 /*IsOrdered*/ true);
2870 return;
2871 case Intrinsic::riscv_vsuxseg2:
2872 case Intrinsic::riscv_vsuxseg3:
2873 case Intrinsic::riscv_vsuxseg4:
2874 case Intrinsic::riscv_vsuxseg5:
2875 case Intrinsic::riscv_vsuxseg6:
2876 case Intrinsic::riscv_vsuxseg7:
2877 case Intrinsic::riscv_vsuxseg8:
2878 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2879 /*IsOrdered*/ false);
2880 return;
2881 case Intrinsic::riscv_vsoxseg2_mask:
2882 case Intrinsic::riscv_vsoxseg3_mask:
2883 case Intrinsic::riscv_vsoxseg4_mask:
2884 case Intrinsic::riscv_vsoxseg5_mask:
2885 case Intrinsic::riscv_vsoxseg6_mask:
2886 case Intrinsic::riscv_vsoxseg7_mask:
2887 case Intrinsic::riscv_vsoxseg8_mask:
2888 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2889 /*IsOrdered*/ true);
2890 return;
2891 case Intrinsic::riscv_vsuxseg2_mask:
2892 case Intrinsic::riscv_vsuxseg3_mask:
2893 case Intrinsic::riscv_vsuxseg4_mask:
2894 case Intrinsic::riscv_vsuxseg5_mask:
2895 case Intrinsic::riscv_vsuxseg6_mask:
2896 case Intrinsic::riscv_vsuxseg7_mask:
2897 case Intrinsic::riscv_vsuxseg8_mask:
2898 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2899 /*IsOrdered*/ false);
2900 return;
2901 case Intrinsic::riscv_vsoxei:
2902 case Intrinsic::riscv_vsoxei_mask:
2903 case Intrinsic::riscv_vsuxei:
2904 case Intrinsic::riscv_vsuxei_mask: {
2905 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
2906 IntNo == Intrinsic::riscv_vsuxei_mask;
2907 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
2908 IntNo == Intrinsic::riscv_vsoxei_mask;
2909
2910 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2911 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2912
2913 unsigned CurOp = 2;
2914 SmallVector<SDValue, 8> Operands;
2915 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2916
2917 MVT IndexVT;
2918 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2919 /*IsStridedOrIndexed*/ true, Operands,
2920 /*IsLoad=*/false, IndexVT: &IndexVT);
2921
2922 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2923 "Element count mismatch");
2924
2925 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2926 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2927 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2928 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2929 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2930 "index values when XLEN=32");
2931 }
2932 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
2933 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW,
2934 LMUL: static_cast<unsigned>(LMUL), IndexLMUL: static_cast<unsigned>(IndexLMUL));
2935 MachineSDNode *Store =
2936 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2937
2938 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2939
2940 ReplaceNode(F: Node, T: Store);
2941 return;
2942 }
2943 case Intrinsic::riscv_vsm:
2944 case Intrinsic::riscv_vse:
2945 case Intrinsic::riscv_vse_mask:
2946 case Intrinsic::riscv_vsse:
2947 case Intrinsic::riscv_vsse_mask: {
2948 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
2949 IntNo == Intrinsic::riscv_vsse_mask;
2950 bool IsStrided =
2951 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
2952
2953 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2954 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2955
2956 unsigned CurOp = 2;
2957 SmallVector<SDValue, 8> Operands;
2958 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2959
2960 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2961 Operands);
2962
2963 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2964 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
2965 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2966 MachineSDNode *Store =
2967 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2968 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2969
2970 ReplaceNode(F: Node, T: Store);
2971 return;
2972 }
2973 case Intrinsic::riscv_sf_vc_x_se:
2974 case Intrinsic::riscv_sf_vc_i_se:
2975 selectSF_VC_X_SE(Node);
2976 return;
2977 case Intrinsic::riscv_sf_vlte8:
2978 case Intrinsic::riscv_sf_vlte16:
2979 case Intrinsic::riscv_sf_vlte32:
2980 case Intrinsic::riscv_sf_vlte64: {
2981 unsigned Log2SEW;
2982 unsigned PseudoInst;
2983 switch (IntNo) {
2984 case Intrinsic::riscv_sf_vlte8:
2985 PseudoInst = RISCV::PseudoSF_VLTE8;
2986 Log2SEW = 3;
2987 break;
2988 case Intrinsic::riscv_sf_vlte16:
2989 PseudoInst = RISCV::PseudoSF_VLTE16;
2990 Log2SEW = 4;
2991 break;
2992 case Intrinsic::riscv_sf_vlte32:
2993 PseudoInst = RISCV::PseudoSF_VLTE32;
2994 Log2SEW = 5;
2995 break;
2996 case Intrinsic::riscv_sf_vlte64:
2997 PseudoInst = RISCV::PseudoSF_VLTE64;
2998 Log2SEW = 6;
2999 break;
3000 }
3001
3002 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
3003 SDValue TWidenOp = CurDAG->getTargetConstant(Val: 1, DL, VT: XLenVT);
3004 SDValue Operands[] = {Node->getOperand(Num: 2),
3005 Node->getOperand(Num: 3),
3006 Node->getOperand(Num: 4),
3007 SEWOp,
3008 TWidenOp,
3009 Node->getOperand(Num: 0)};
3010
3011 MachineSDNode *TileLoad =
3012 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
3013 CurDAG->setNodeMemRefs(N: TileLoad,
3014 NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
3015
3016 ReplaceNode(F: Node, T: TileLoad);
3017 return;
3018 }
3019 case Intrinsic::riscv_sf_mm_s_s:
3020 case Intrinsic::riscv_sf_mm_s_u:
3021 case Intrinsic::riscv_sf_mm_u_s:
3022 case Intrinsic::riscv_sf_mm_u_u:
3023 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
3024 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
3025 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
3026 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
3027 case Intrinsic::riscv_sf_mm_f_f: {
3028 bool HasFRM = false;
3029 unsigned PseudoInst;
3030 switch (IntNo) {
3031 case Intrinsic::riscv_sf_mm_s_s:
3032 PseudoInst = RISCV::PseudoSF_MM_S_S;
3033 break;
3034 case Intrinsic::riscv_sf_mm_s_u:
3035 PseudoInst = RISCV::PseudoSF_MM_S_U;
3036 break;
3037 case Intrinsic::riscv_sf_mm_u_s:
3038 PseudoInst = RISCV::PseudoSF_MM_U_S;
3039 break;
3040 case Intrinsic::riscv_sf_mm_u_u:
3041 PseudoInst = RISCV::PseudoSF_MM_U_U;
3042 break;
3043 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
3044 PseudoInst = RISCV::PseudoSF_MM_E5M2_E5M2;
3045 HasFRM = true;
3046 break;
3047 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
3048 PseudoInst = RISCV::PseudoSF_MM_E5M2_E4M3;
3049 HasFRM = true;
3050 break;
3051 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
3052 PseudoInst = RISCV::PseudoSF_MM_E4M3_E5M2;
3053 HasFRM = true;
3054 break;
3055 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
3056 PseudoInst = RISCV::PseudoSF_MM_E4M3_E4M3;
3057 HasFRM = true;
3058 break;
3059 case Intrinsic::riscv_sf_mm_f_f:
3060 if (Node->getOperand(Num: 3).getValueType().getScalarType() == MVT::bf16)
3061 PseudoInst = RISCV::PseudoSF_MM_F_F_ALT;
3062 else
3063 PseudoInst = RISCV::PseudoSF_MM_F_F;
3064 HasFRM = true;
3065 break;
3066 }
3067 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3068 SDValue Op1 = Node->getOperand(Num: 3);
3069 SDValue Op2 = Node->getOperand(Num: 4);
3070 MVT VT = Op1->getSimpleValueType(ResNo: 0);
3071 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3072 SDValue TmOp = Node->getOperand(Num: 5);
3073 SDValue TnOp = Node->getOperand(Num: 6);
3074 SDValue TkOp = Node->getOperand(Num: 7);
3075 SDValue TWidenOp = Node->getOperand(Num: 8);
3076 SDValue Chain = Node->getOperand(Num: 0);
3077
3078 // sf.mm.f.f with sew=32, twiden=2 is invalid
3079 if (IntNo == Intrinsic::riscv_sf_mm_f_f && Log2SEW == 5 &&
3080 TWidenOp->getAsZExtVal() == 2)
3081 reportFatalUsageError(reason: "sf.mm.f.f doesn't support (sew=32, twiden=2)");
3082
3083 SmallVector<SDValue, 10> Operands(
3084 {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Op1, Op2});
3085 if (HasFRM)
3086 Operands.push_back(
3087 Elt: CurDAG->getTargetConstant(Val: RISCVFPRndMode::DYN, DL, VT: XLenVT));
3088 Operands.append(IL: {TmOp, TnOp, TkOp,
3089 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT), TWidenOp,
3090 Chain});
3091
3092 auto *NewNode =
3093 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
3094
3095 ReplaceNode(F: Node, T: NewNode);
3096 return;
3097 }
3098 case Intrinsic::riscv_sf_vtzero_t: {
3099 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3100 SDValue Tm = Node->getOperand(Num: 3);
3101 SDValue Tn = Node->getOperand(Num: 4);
3102 SDValue Log2SEW = Node->getOperand(Num: 5);
3103 SDValue TWiden = Node->getOperand(Num: 6);
3104 SDValue Chain = Node->getOperand(Num: 0);
3105 auto *NewNode = CurDAG->getMachineNode(
3106 Opcode: RISCV::PseudoSF_VTZERO_T, dl: DL, VTs: Node->getVTList(),
3107 Ops: {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Tm, Tn, Log2SEW,
3108 TWiden, Chain});
3109
3110 ReplaceNode(F: Node, T: NewNode);
3111 return;
3112 }
3113 }
3114 break;
3115 }
3116 case ISD::BITCAST: {
3117 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3118 // Just drop bitcasts between vectors if both are fixed or both are
3119 // scalable.
3120 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
3121 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
3122 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3123 CurDAG->RemoveDeadNode(N: Node);
3124 return;
3125 }
3126 if (Subtarget->hasStdExtP()) {
3127 bool Is32BitCast =
3128 (VT == MVT::i32 && (SrcVT == MVT::v4i8 || SrcVT == MVT::v2i16)) ||
3129 (SrcVT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
3130 bool Is64BitCast =
3131 (VT == MVT::i64 && (SrcVT == MVT::v8i8 || SrcVT == MVT::v4i16 ||
3132 SrcVT == MVT::v2i32)) ||
3133 (SrcVT == MVT::i64 &&
3134 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
3135 if (Is32BitCast || Is64BitCast) {
3136 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3137 CurDAG->RemoveDeadNode(N: Node);
3138 return;
3139 }
3140 }
3141 break;
3142 }
3143 case ISD::SPLAT_VECTOR: {
3144 if (!Subtarget->hasStdExtP())
3145 break;
3146 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0))) {
3147 bool IsDoubleWide = Subtarget->isPExtPackedDoubleType(VT);
3148
3149 if (ConstNode->isZero()) {
3150 MCPhysReg X0Reg = IsDoubleWide ? RISCV::X0_Pair : RISCV::X0;
3151 SDValue New =
3152 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: X0Reg, VT);
3153 ReplaceNode(F: Node, T: New.getNode());
3154 return;
3155 }
3156
3157 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3158 APInt Val = ConstNode->getAPIntValue().trunc(width: EltSize);
3159
3160 // Use LI for all ones since it can be compressed to c.li.
3161 if (Val.isAllOnes() && !IsDoubleWide) {
3162 SDNode *NewNode = CurDAG->getMachineNode(
3163 Opcode: RISCV::ADDI, dl: DL, VT, Op1: CurDAG->getRegister(Reg: RISCV::X0, VT),
3164 Op2: CurDAG->getAllOnesConstant(DL, VT: XLenVT, /*IsTarget=*/true));
3165 ReplaceNode(F: Node, T: NewNode);
3166 return;
3167 }
3168
3169 // Find the smallest splat.
3170 if (Val.getBitWidth() > 16 && Val.isSplat(SplatSizeInBits: 16))
3171 Val = Val.trunc(width: 16);
3172 if (Val.getBitWidth() > 8 && Val.isSplat(SplatSizeInBits: 8))
3173 Val = Val.trunc(width: 8);
3174
3175 EltSize = Val.getBitWidth();
3176 int64_t Imm = Val.getSExtValue();
3177
3178 unsigned Opc = 0;
3179 if (EltSize == 8) {
3180 Opc = IsDoubleWide ? RISCV::PLI_DB : RISCV::PLI_B;
3181 } else if (EltSize == 16 && isInt<10>(x: Imm)) {
3182 Opc = IsDoubleWide ? RISCV::PLI_DH : RISCV::PLI_H;
3183 } else if (!IsDoubleWide && EltSize == 32 && isInt<10>(x: Imm)) {
3184 Opc = RISCV::PLI_W;
3185 } else if (EltSize == 16 && isShiftedInt<10, 6>(x: Imm)) {
3186 Opc = IsDoubleWide ? RISCV::PLUI_DH : RISCV::PLUI_H;
3187 Imm = Imm >> 6;
3188 } else if (!IsDoubleWide && EltSize == 32 && isShiftedInt<10, 22>(x: Imm)) {
3189 Opc = RISCV::PLUI_W;
3190 Imm = Imm >> 22;
3191 }
3192
3193 if (Opc) {
3194 SDNode *NewNode = CurDAG->getMachineNode(
3195 Opcode: Opc, dl: DL, VT, Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT: XLenVT));
3196 ReplaceNode(F: Node, T: NewNode);
3197 return;
3198 }
3199 }
3200
3201 break;
3202 }
3203 case ISD::SCALAR_TO_VECTOR:
3204 if (Subtarget->hasStdExtP()) {
3205 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3206 if (SrcVT == Subtarget->getXLenVT() &&
3207 VT.getSizeInBits() == SrcVT.getSizeInBits()) {
3208 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3209 CurDAG->RemoveDeadNode(N: Node);
3210 return;
3211 }
3212 }
3213 break;
3214 case ISD::INSERT_SUBVECTOR:
3215 case RISCVISD::TUPLE_INSERT: {
3216 SDValue V = Node->getOperand(Num: 0);
3217 SDValue SubV = Node->getOperand(Num: 1);
3218 SDLoc DL(SubV);
3219 auto Idx = Node->getConstantOperandVal(Num: 2);
3220 MVT SubVecVT = SubV.getSimpleValueType();
3221
3222 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3223 MVT SubVecContainerVT = SubVecVT;
3224 // Establish the correct scalable-vector types for any fixed-length type.
3225 if (SubVecVT.isFixedLengthVector()) {
3226 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT: SubVecVT);
3227 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
3228 [[maybe_unused]] bool ExactlyVecRegSized =
3229 Subtarget->expandVScale(X: SubVecVT.getSizeInBits())
3230 .isKnownMultipleOf(RHS: Subtarget->expandVScale(X: VecRegSize));
3231 assert(isPowerOf2_64(Subtarget->expandVScale(SubVecVT.getSizeInBits())
3232 .getKnownMinValue()));
3233 assert(Idx == 0 && (ExactlyVecRegSized || V.isUndef()));
3234 }
3235 MVT ContainerVT = VT;
3236 if (VT.isFixedLengthVector())
3237 ContainerVT = TLI.getContainerForFixedLengthVector(VT);
3238
3239 const auto *TRI = Subtarget->getRegisterInfo();
3240 unsigned SubRegIdx;
3241 std::tie(args&: SubRegIdx, args&: Idx) =
3242 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3243 VecVT: ContainerVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3244
3245 // If the Idx hasn't been completely eliminated then this is a subvector
3246 // insert which doesn't naturally align to a vector register. These must
3247 // be handled using instructions to manipulate the vector registers.
3248 if (Idx != 0)
3249 break;
3250
3251 RISCVVType::VLMUL SubVecLMUL =
3252 RISCVTargetLowering::getLMUL(VT: SubVecContainerVT);
3253 [[maybe_unused]] bool IsSubVecPartReg =
3254 SubVecLMUL == RISCVVType::VLMUL::LMUL_F2 ||
3255 SubVecLMUL == RISCVVType::VLMUL::LMUL_F4 ||
3256 SubVecLMUL == RISCVVType::VLMUL::LMUL_F8;
3257 assert((V.getValueType().isRISCVVectorTuple() || !IsSubVecPartReg ||
3258 V.isUndef()) &&
3259 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
3260 "the subvector is smaller than a full-sized register");
3261
3262 // If we haven't set a SubRegIdx, then we must be going between
3263 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
3264 if (SubRegIdx == RISCV::NoSubRegister) {
3265 unsigned InRegClassID =
3266 RISCVTargetLowering::getRegClassIDForVecVT(VT: ContainerVT);
3267 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3268 InRegClassID &&
3269 "Unexpected subvector extraction");
3270 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3271 SDNode *NewNode = CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
3272 dl: DL, VT, Op1: SubV, Op2: RC);
3273 ReplaceNode(F: Node, T: NewNode);
3274 return;
3275 }
3276
3277 SDValue Insert = CurDAG->getTargetInsertSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V, Subreg: SubV);
3278 ReplaceNode(F: Node, T: Insert.getNode());
3279 return;
3280 }
3281 case RISCVISD::TUPLE_CAST: {
3282 // TUPLE_CAST reinterprets a vector tuple as a different tuple type with
3283 // the same register class (same NF and LMUL), just a different minimum
3284 // number of elements per field. The two types occupy identical
3285 // registers, so just forward the operand like we do for same-register-
3286 // class ISD::BITCAST, without emitting a copy.
3287 assert(RISCVTargetLowering::getRegClassIDForVecVT(VT) ==
3288 RISCVTargetLowering::getRegClassIDForVecVT(
3289 Node->getOperand(0).getSimpleValueType()) &&
3290 "Expected input and output of TUPLE_CAST to use the same "
3291 "register class");
3292 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3293 CurDAG->RemoveDeadNode(N: Node);
3294 return;
3295 }
3296 case ISD::EXTRACT_SUBVECTOR:
3297 case RISCVISD::TUPLE_EXTRACT: {
3298 if (Subtarget->hasStdExtP())
3299 break;
3300
3301 SDValue V = Node->getOperand(Num: 0);
3302 auto Idx = Node->getConstantOperandVal(Num: 1);
3303 MVT InVT = V.getSimpleValueType();
3304
3305 SDLoc DL(V);
3306
3307 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3308 MVT SubVecContainerVT = VT;
3309 // Establish the correct scalable-vector types for any fixed-length type.
3310 if (VT.isFixedLengthVector()) {
3311 assert(Idx == 0);
3312 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
3313 }
3314 if (InVT.isFixedLengthVector())
3315 InVT = TLI.getContainerForFixedLengthVector(VT: InVT);
3316
3317 const auto *TRI = Subtarget->getRegisterInfo();
3318 unsigned SubRegIdx;
3319 std::tie(args&: SubRegIdx, args&: Idx) =
3320 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3321 VecVT: InVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3322
3323 // If the Idx hasn't been completely eliminated then this is a subvector
3324 // extract which doesn't naturally align to a vector register. These must
3325 // be handled using instructions to manipulate the vector registers.
3326 if (Idx != 0)
3327 break;
3328
3329 // If we haven't set a SubRegIdx, then we must be going between
3330 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
3331 if (SubRegIdx == RISCV::NoSubRegister) {
3332 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: InVT);
3333 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3334 InRegClassID &&
3335 "Unexpected subvector extraction");
3336 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3337 SDNode *NewNode =
3338 CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT, Op1: V, Op2: RC);
3339 ReplaceNode(F: Node, T: NewNode);
3340 return;
3341 }
3342
3343 SDValue Extract = CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V);
3344 ReplaceNode(F: Node, T: Extract.getNode());
3345 return;
3346 }
3347 case RISCVISD::VMV_S_X_VL:
3348 case RISCVISD::VFMV_S_F_VL:
3349 case RISCVISD::VMV_V_X_VL:
3350 case RISCVISD::VFMV_V_F_VL: {
3351 // Try to match splat of a scalar load to a strided load with stride of x0.
3352 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
3353 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
3354 if (!Node->getOperand(Num: 0).isUndef())
3355 break;
3356 SDValue Src = Node->getOperand(Num: 1);
3357 auto *Ld = dyn_cast<LoadSDNode>(Val&: Src);
3358 // Can't fold load update node because the second
3359 // output is used so that load update node can't be removed.
3360 if (!Ld || Ld->isIndexed())
3361 break;
3362 EVT MemVT = Ld->getMemoryVT();
3363 // The memory VT should be the same size as the element type.
3364 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
3365 break;
3366 if (!IsProfitableToFold(N: Src, U: Node, Root: Node) ||
3367 !IsLegalToFold(N: Src, U: Node, Root: Node, OptLevel: TM.getOptLevel()))
3368 break;
3369
3370 SDValue VL;
3371 if (IsScalarMove) {
3372 // We could deal with more VL if we update the VSETVLI insert pass to
3373 // avoid introducing more VSETVLI.
3374 if (!isOneConstant(V: Node->getOperand(Num: 2)))
3375 break;
3376 selectVLOp(N: Node->getOperand(Num: 2), VL);
3377 } else
3378 selectVLOp(N: Node->getOperand(Num: 2), VL);
3379
3380 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3381 SDValue SEW = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
3382
3383 // If VL=1, then we don't need to do a strided load and can just do a
3384 // regular load.
3385 bool IsStrided = !isOneConstant(V: VL);
3386
3387 // Only do a strided load if we have optimized zero-stride vector load.
3388 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
3389 break;
3390
3391 SmallVector<SDValue> Operands = {
3392 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT), 0),
3393 Ld->getBasePtr()};
3394 if (IsStrided)
3395 Operands.push_back(Elt: CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT));
3396 uint64_t Policy = RISCVVType::MASK_AGNOSTIC | RISCVVType::TAIL_AGNOSTIC;
3397 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
3398 Operands.append(IL: {VL, SEW, PolicyOp, Ld->getChain()});
3399
3400 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
3401 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
3402 /*IsMasked*/ Masked: false, Strided: IsStrided, /*FF*/ false,
3403 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
3404 MachineSDNode *Load =
3405 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, ResultTys: {VT, MVT::Other}, Ops: Operands);
3406 // Update the chain.
3407 ReplaceUses(F: Src.getValue(R: 1), T: SDValue(Load, 1));
3408 // Record the mem-refs
3409 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {Ld->getMemOperand()});
3410 // Replace the splat with the vlse.
3411 ReplaceNode(F: Node, T: Load);
3412 return;
3413 }
3414 case RISCVISD::LPAD_CALL:
3415 case RISCVISD::LPAD_CALL_INDIRECT: {
3416 bool IsIndirect = Opcode == RISCVISD::LPAD_CALL_INDIRECT;
3417 unsigned PseudoOpc = IsIndirect ? RISCV::PseudoCALLIndirectLpadAlign
3418 : RISCV::PseudoCALLLpadAlign;
3419
3420 uint32_t LpadLabel = 0;
3421 if (std::optional<uint32_t> Label =
3422 Subtarget->getCLOpts().landing_pad_label) {
3423 if (!isUInt<20>(x: *Label))
3424 report_fatal_error(reason: "riscv-landing-pad-label=<val>, <val> needs to fit "
3425 "in unsigned 20-bits");
3426 LpadLabel = *Label;
3427 }
3428
3429 // Preserve the argument-register and register-mask operands, between
3430 // Callee and the optional glue, so the pseudo call still reports its
3431 // call-preserved mask to the register allocator.
3432 SmallVector<SDValue, 8> Ops;
3433 Ops.push_back(Elt: Node->getOperand(Num: 1));
3434 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: LpadLabel, DL, VT: XLenVT));
3435
3436 unsigned NumOps = Node->getNumOperands();
3437 bool HasGlue = Node->getGluedNode() != nullptr;
3438 unsigned RegOperandsEnd = HasGlue ? NumOps - 1 : NumOps;
3439 for (unsigned I = 2; I != RegOperandsEnd; ++I)
3440 Ops.push_back(Elt: Node->getOperand(Num: I));
3441
3442 Ops.push_back(Elt: Node->getOperand(Num: 0));
3443 if (HasGlue)
3444 Ops.push_back(Elt: Node->getOperand(Num: NumOps - 1));
3445
3446 ReplaceNode(F: Node,
3447 T: CurDAG->getMachineNode(Opcode: PseudoOpc, dl: DL, VTs: Node->getVTList(), Ops));
3448 return;
3449 }
3450 case ISD::PREFETCH:
3451 // MIPS's prefetch instruction already encodes the hint within the
3452 // instruction itself, so no extra NTL hint is needed.
3453 if (Subtarget->hasVendorXMIPSCBOP())
3454 break;
3455
3456 unsigned Locality = Node->getConstantOperandVal(Num: 3);
3457 if (Locality > 2)
3458 break;
3459
3460 auto *LoadStoreMem = cast<MemSDNode>(Val: Node);
3461 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
3462 MMO->setFlags(MachineMemOperand::MONonTemporal);
3463
3464 int NontemporalLevel = 0;
3465 switch (Locality) {
3466 case 0:
3467 NontemporalLevel = 3; // NTL.ALL
3468 break;
3469 case 1:
3470 NontemporalLevel = 1; // NTL.PALL
3471 break;
3472 case 2:
3473 NontemporalLevel = 0; // NTL.P1
3474 break;
3475 default:
3476 llvm_unreachable("unexpected locality value.");
3477 }
3478
3479 if (NontemporalLevel & 0b1)
3480 MMO->setFlags(MONontemporalBit0);
3481 if (NontemporalLevel & 0b10)
3482 MMO->setFlags(MONontemporalBit1);
3483 break;
3484 }
3485
3486 // Select the default instruction.
3487 SelectCode(N: Node);
3488}
3489
3490bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
3491 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
3492 std::vector<SDValue> &OutOps) {
3493 // Always produce a register and immediate operand, as expected by
3494 // RISCVAsmPrinter::PrintAsmMemoryOperand.
3495 switch (ConstraintID) {
3496 case InlineAsm::ConstraintCode::o:
3497 case InlineAsm::ConstraintCode::m: {
3498 SDValue Op0, Op1;
3499 [[maybe_unused]] bool Found = SelectAddrRegImm(Addr: Op, Base&: Op0, Offset&: Op1);
3500 assert(Found && "SelectAddrRegImm should always succeed");
3501 OutOps.push_back(x: Op0);
3502 OutOps.push_back(x: Op1);
3503 return false;
3504 }
3505 case InlineAsm::ConstraintCode::A:
3506 OutOps.push_back(x: Op);
3507 OutOps.push_back(
3508 x: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget->getXLenVT()));
3509 return false;
3510 default:
3511 report_fatal_error(reason: "Unexpected asm memory constraint " +
3512 InlineAsm::getMemConstraintName(C: ConstraintID));
3513 }
3514
3515 return true;
3516}
3517
3518bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
3519 SDValue &Offset) {
3520 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Addr)) {
3521 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT: Subtarget->getXLenVT());
3522 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: Subtarget->getXLenVT());
3523 return true;
3524 }
3525
3526 return false;
3527}
3528
3529// Fold constant addresses.
3530static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
3531 const MVT VT, const RISCVSubtarget *Subtarget,
3532 SDValue Addr, SDValue &Base, SDValue &Offset,
3533 bool IsPrefetch = false) {
3534 if (!isa<ConstantSDNode>(Val: Addr))
3535 return false;
3536
3537 int64_t CVal = cast<ConstantSDNode>(Val&: Addr)->getSExtValue();
3538
3539 // If the constant is a simm12, we can fold the whole constant and use X0 as
3540 // the base. If the constant can be materialized with LUI+simm12, use LUI as
3541 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
3542 int64_t Lo12 = SignExtend64<12>(x: CVal);
3543 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
3544 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
3545 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3546 return false;
3547 if (Hi) {
3548 int64_t Hi20 = (Hi >> 12) & 0xfffff;
3549 Base = SDValue(
3550 CurDAG->getMachineNode(Opcode: RISCV::LUI, dl: DL, VT,
3551 Op1: CurDAG->getTargetConstant(Val: Hi20, DL, VT)),
3552 0);
3553 } else {
3554 Base = CurDAG->getRegister(Reg: RISCV::X0, VT);
3555 }
3556 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3557 return true;
3558 }
3559
3560 // Ask how constant materialization would handle this constant.
3561 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
3562
3563 // If the last instruction would be an ADDI, we can fold its immediate and
3564 // emit the rest of the sequence as the base.
3565 if (Seq.back().getOpcode() != RISCV::ADDI)
3566 return false;
3567 Lo12 = Seq.back().getImm();
3568 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3569 return false;
3570
3571 // Drop the last instruction.
3572 Seq.pop_back();
3573 assert(!Seq.empty() && "Expected more instructions in sequence");
3574
3575 Base = selectImmSeq(CurDAG, DL, VT, Seq);
3576 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3577 return true;
3578}
3579
3580// Is this ADD instruction only used as the base pointer of scalar loads and
3581// stores?
3582static bool isWorthFoldingAdd(SDValue Add) {
3583 for (auto *User : Add->users()) {
3584 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
3585 User->getOpcode() != RISCVISD::LD_RV32 &&
3586 User->getOpcode() != RISCVISD::SD_RV32 &&
3587 User->getOpcode() != ISD::ATOMIC_LOAD &&
3588 User->getOpcode() != ISD::ATOMIC_STORE)
3589 return false;
3590 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3591 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
3592 VT != MVT::f64)
3593 return false;
3594 // Don't allow stores of the value. It must be used as the address.
3595 if (User->getOpcode() == ISD::STORE &&
3596 cast<StoreSDNode>(Val: User)->getValue() == Add)
3597 return false;
3598 if (User->getOpcode() == ISD::ATOMIC_STORE &&
3599 cast<AtomicSDNode>(Val: User)->getVal() == Add)
3600 return false;
3601 if (User->getOpcode() == RISCVISD::SD_RV32 &&
3602 (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add))
3603 return false;
3604 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
3605 return false;
3606 }
3607
3608 return true;
3609}
3610
3611bool isRegImmLoadOrStore(SDNode *User, SDValue Add) {
3612 switch (User->getOpcode()) {
3613 default:
3614 return false;
3615 case ISD::LOAD:
3616 case RISCVISD::LD_RV32:
3617 case ISD::ATOMIC_LOAD:
3618 break;
3619 case ISD::STORE:
3620 // Don't allow stores of Add. It must only be used as the address.
3621 if (cast<StoreSDNode>(Val: User)->getValue() == Add)
3622 return false;
3623 break;
3624 case RISCVISD::SD_RV32:
3625 // Don't allow stores of Add. It must only be used as the address.
3626 if (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add)
3627 return false;
3628 break;
3629 case ISD::ATOMIC_STORE:
3630 // Don't allow stores of Add. It must only be used as the address.
3631 if (cast<AtomicSDNode>(Val: User)->getVal() == Add)
3632 return false;
3633 break;
3634 }
3635
3636 return true;
3637}
3638
3639// To prevent SelectAddrRegImm from folding offsets that conflict with the
3640// fusion of PseudoMovAddr, check if the offset of every use of a given address
3641// is within the alignment.
3642bool RISCVDAGToDAGISel::areOffsetsWithinAlignment(SDValue Addr,
3643 Align Alignment) {
3644 assert(Addr->getOpcode() == RISCVISD::ADD_LO);
3645 for (auto *User : Addr->users()) {
3646 // If the user is a load or store, then the offset is 0 which is always
3647 // within alignment.
3648 if (isRegImmLoadOrStore(User, Add: Addr))
3649 continue;
3650
3651 if (CurDAG->isBaseWithConstantOffset(Op: SDValue(User, 0))) {
3652 int64_t CVal = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
3653 if (!isInt<12>(x: CVal) || Alignment <= CVal)
3654 return false;
3655
3656 // Make sure all uses are foldable load/stores.
3657 for (auto *AddUser : User->users())
3658 if (!isRegImmLoadOrStore(User: AddUser, Add: SDValue(User, 0)))
3659 return false;
3660
3661 continue;
3662 }
3663
3664 return false;
3665 }
3666
3667 return true;
3668}
3669
3670bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
3671 SDValue &Offset) {
3672 if (SelectAddrFrameIndex(Addr, Base, Offset))
3673 return true;
3674
3675 SDLoc DL(Addr);
3676 MVT VT = Addr.getSimpleValueType();
3677
3678 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
3679 bool CanFold = true;
3680 // Unconditionally fold if operand 1 is not a global address (e.g.
3681 // externsymbol)
3682 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val: Addr.getOperand(i: 1))) {
3683 const DataLayout &DL = CurDAG->getDataLayout();
3684 Align Alignment = commonAlignment(
3685 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3686 if (!areOffsetsWithinAlignment(Addr, Alignment))
3687 CanFold = false;
3688 }
3689 if (CanFold) {
3690 Base = Addr.getOperand(i: 0);
3691 Offset = Addr.getOperand(i: 1);
3692 return true;
3693 }
3694 }
3695
3696 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3697 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3698 if (isInt<12>(x: CVal)) {
3699 Base = Addr.getOperand(i: 0);
3700 if (Base.getOpcode() == RISCVISD::ADD_LO) {
3701 SDValue LoOperand = Base.getOperand(i: 1);
3702 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: LoOperand)) {
3703 // If the Lo in (ADD_LO hi, lo) is a global variable's address
3704 // (its low part, really), then we can rely on the alignment of that
3705 // variable to provide a margin of safety before low part can overflow
3706 // the 12 bits of the load/store offset. Check if CVal falls within
3707 // that margin; if so (low part + CVal) can't overflow.
3708 const DataLayout &DL = CurDAG->getDataLayout();
3709 Align Alignment = commonAlignment(
3710 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3711 if ((CVal == 0 || Alignment > CVal) &&
3712 areOffsetsWithinAlignment(Addr: Base, Alignment)) {
3713 int64_t CombinedOffset = CVal + GA->getOffset();
3714 Base = Base.getOperand(i: 0);
3715 Offset = CurDAG->getTargetGlobalAddress(
3716 GV: GA->getGlobal(), DL: SDLoc(LoOperand), VT: LoOperand.getValueType(),
3717 offset: CombinedOffset, TargetFlags: GA->getTargetFlags());
3718 return true;
3719 }
3720 }
3721 }
3722
3723 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3724 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3725 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3726 return true;
3727 }
3728 }
3729
3730 // Handle ADD with large immediates.
3731 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3732 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3733 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3734
3735 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
3736 // an ADDI for part of the offset and fold the rest into the load/store.
3737 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
3738 if (CVal >= -4096 && CVal <= 4094) {
3739 int64_t Adj = CVal < 0 ? -2048 : 2047;
3740 Base = SDValue(
3741 CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3742 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3743 0);
3744 Offset = CurDAG->getSignedTargetConstant(Val: CVal - Adj, DL, VT);
3745 return true;
3746 }
3747
3748 // For larger immediates, we might be able to save one instruction from
3749 // constant materialization by folding the Lo12 bits of the immediate into
3750 // the address. We should only do this if the ADD is only used by loads and
3751 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
3752 // separately with the full materialized immediate creating extra
3753 // instructions.
3754 if (isWorthFoldingAdd(Add: Addr) &&
3755 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3756 Offset, /*IsPrefetch=*/false)) {
3757 // Insert an ADD instruction with the materialized Hi52 bits.
3758 Base = SDValue(
3759 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3760 0);
3761 return true;
3762 }
3763 }
3764
3765 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3766 /*IsPrefetch=*/false))
3767 return true;
3768
3769 Base = Addr;
3770 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3771 return true;
3772}
3773
3774/// Similar to SelectAddrRegImm, but only matches a register, or a register
3775/// plus a simm12 offset. Doesn't match a FrameIndex or global address, since
3776/// those aren't valid for the callers of this function (e.g. the target of
3777/// an indirect branch).
3778bool RISCVDAGToDAGISel::SelectBrindRegImm(SDValue Addr, SDValue &Base,
3779 SDValue &Offset) {
3780 SDLoc DL(Addr);
3781 MVT VT = Addr.getSimpleValueType();
3782
3783 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3784 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3785 if (isInt<12>(x: CVal)) {
3786 Base = Addr.getOperand(i: 0);
3787 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3788 return true;
3789 }
3790 }
3791
3792 Base = Addr;
3793 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3794 return true;
3795}
3796
3797/// Similar to SelectAddrRegImm, except that the offset is a 26-bit signed
3798/// immediate. This is used by the Qualcomm Xqcilo large offset load/store
3799/// instructions (qc.e.lw/qc.e.sw), whose offset field is 26 bits wide.
3800/// Only matches offsets that do not fit a 12-bit signed immediate, so that
3801/// offsets in the simm12 range keep using the shorter (and possibly
3802/// compressible) standard load/store instructions.
3803bool RISCVDAGToDAGISel::SelectAddrRegImm26(SDValue Addr, SDValue &Base,
3804 SDValue &Offset) {
3805 SDLoc DL(Addr);
3806 MVT VT = Addr.getSimpleValueType();
3807
3808 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3809 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3810 // Fold a 26-bit (but not 12-bit) signed offset directly into the
3811 // load/store.
3812 if (isInt<26>(x: CVal) && !isInt<12>(x: CVal)) {
3813 Base = Addr.getOperand(i: 0);
3814 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3815 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3816 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3817 return true;
3818 }
3819 }
3820
3821 // The offset is just outside the 26-bit range. Split off a small (simm12)
3822 // adjustment with a plain ADDI and fold the remaining 26-bit offset into the
3823 // load/store. A plain ADDI is used (rather than the wide
3824 // qc.e.addi/qc.e.addai) because the adjustment fits simm12: this keeps it a
3825 // short, compressible (c.addi) instruction and is available without Xqcilia.
3826 //
3827 // Skip the split if the address is used other than as a foldable load/store
3828 // base. `isWorthFoldingAdd()` returns true when every user of the add node is
3829 // a scalar load/store using it as an address operand. If it return false, it
3830 // means that some use consumes the add result as a value (e.g. it feeds
3831 // another add, is a stored value, is used in arithmetic) and that use forces
3832 // the add to be materialized into a register.
3833 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)) &&
3834 isWorthFoldingAdd(Add: Addr)) {
3835 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3836 if (!isInt<26>(x: CVal)) {
3837 // check if lw in lui + add + lw combination can be compressed.
3838 // The check here purely based on the immediate value and hopes that
3839 // register allocator would assign a register from a GPRC set so that the
3840 // instruction can get compressed.
3841 bool IsLwCompressable = isShiftedUInt<5, 2>(x: CVal & ((1 << 12) - 1));
3842
3843 int64_t Imm26 = CVal < 0 ? minIntN(N: 26) : maxIntN(N: 26);
3844 int64_t Adj = CVal - Imm26;
3845 // If Adj fits within 6-bits, then both combinations will take 8 bytes
3846 // however c.addi + qc.e.lw/sw will take 1 less cycle. Also, if lw is not
3847 // compressable then both combination would take 10 bytes but again
3848 // addi + qc.e.lw/sw will take 1 less cycle.
3849 if (isInt<6>(x: Adj) || (isInt<12>(x: Adj) && !IsLwCompressable)) {
3850 Base = SDValue(CurDAG->getMachineNode(
3851 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3852 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3853 0);
3854 Offset = CurDAG->getSignedTargetConstant(Val: Imm26, DL, VT);
3855 return true;
3856 }
3857 }
3858 }
3859
3860 // Don't match: let the standard addressing modes handle it.
3861 return false;
3862}
3863
3864/// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
3865bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
3866 SDValue &Offset) {
3867 if (SelectAddrFrameIndex(Addr, Base, Offset))
3868 return true;
3869
3870 SDLoc DL(Addr);
3871 MVT VT = Addr.getSimpleValueType();
3872
3873 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3874 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3875 if (isUInt<9>(x: CVal)) {
3876 Base = Addr.getOperand(i: 0);
3877
3878 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3879 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3880 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3881 return true;
3882 }
3883 }
3884
3885 Base = Addr;
3886 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3887 return true;
3888}
3889
3890/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
3891/// Offset should be all zeros.
3892bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
3893 SDValue &Offset) {
3894 if (SelectAddrFrameIndex(Addr, Base, Offset))
3895 return true;
3896
3897 SDLoc DL(Addr);
3898 MVT VT = Addr.getSimpleValueType();
3899
3900 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3901 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3902 if (isInt<12>(x: CVal)) {
3903 Base = Addr.getOperand(i: 0);
3904
3905 // Early-out if not a valid offset.
3906 if ((CVal & 0b11111) != 0) {
3907 Base = Addr;
3908 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3909 return true;
3910 }
3911
3912 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3913 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3914 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3915 return true;
3916 }
3917 }
3918
3919 // Handle ADD with large immediates.
3920 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3921 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3922 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3923
3924 // Handle immediates in the range [-4096,-2049] or [2017, 4063]. We can save
3925 // one instruction by folding adjustment (-2048 or 2016) into the address.
3926 // The upper bound keeps CVal - 2016 within simm12 ([−2048, 2047]).
3927 if ((-2049 >= CVal && CVal >= -4096) || (4063 >= CVal && CVal >= 2017)) {
3928 int64_t Adj = CVal < 0 ? -2048 : 2016;
3929 int64_t AdjustedOffset = CVal - Adj;
3930 Base =
3931 SDValue(CurDAG->getMachineNode(
3932 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3933 Op2: CurDAG->getSignedTargetConstant(Val: AdjustedOffset, DL, VT)),
3934 0);
3935 Offset = CurDAG->getSignedTargetConstant(Val: Adj, DL, VT);
3936 return true;
3937 }
3938
3939 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3940 Offset, /*IsPrefetch=*/true)) {
3941 // Insert an ADD instruction with the materialized Hi52 bits.
3942 Base = SDValue(
3943 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3944 0);
3945 return true;
3946 }
3947 }
3948
3949 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3950 /*IsPrefetch=*/true))
3951 return true;
3952
3953 Base = Addr;
3954 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3955 return true;
3956}
3957
3958/// Return true if this a load/store that we have a RegRegScale instruction for.
3959static bool isRegRegScaleLoadOrStore(SDNode *User, SDValue Add,
3960 const RISCVSubtarget &Subtarget) {
3961 unsigned UserOpc = User->getOpcode();
3962 if (UserOpc != ISD::LOAD && UserOpc != ISD::STORE)
3963 return false;
3964 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3965 // Zilx only provides indexed loads, so it must not enable reg+reg-scale
3966 // address folding for stores. XTheadMemIdx and Xqcisls have scaled stores.
3967 bool HasScalarIntegerMemIdx =
3968 Subtarget.hasVendorXTHeadMemIdx() || Subtarget.hasVendorXqcisls() ||
3969 (Subtarget.hasStdExtZilx() && UserOpc == ISD::LOAD);
3970 if (!(VT.isScalarInteger() && HasScalarIntegerMemIdx) &&
3971 !((VT == MVT::f32 || VT == MVT::f64) &&
3972 Subtarget.hasVendorXTHeadFMemIdx()))
3973 return false;
3974 // Don't allow stores of the value. It must be used as the address.
3975 if (UserOpc == ISD::STORE && cast<StoreSDNode>(Val: User)->getValue() == Add)
3976 return false;
3977
3978 return true;
3979}
3980
3981/// Is it profitable to fold this Add into RegRegScale load/store. If \p
3982/// Shift is non-null, then we have matched a shl+add. We allow reassociating
3983/// (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2)) if there is a
3984/// single addi and we don't have a SHXADD instruction we could use.
3985/// FIXME: May still need to check how many and what kind of users the SHL has.
3986static bool isWorthFoldingIntoRegRegScale(const RISCVSubtarget &Subtarget,
3987 SDValue Add,
3988 SDValue Shift = SDValue()) {
3989 bool FoundADDI = false;
3990 for (auto *User : Add->users()) {
3991 if (isRegRegScaleLoadOrStore(User, Add, Subtarget))
3992 continue;
3993
3994 // Allow a single ADDI that is used by loads/stores if we matched a shift.
3995 if (!Shift || FoundADDI || User->getOpcode() != ISD::ADD ||
3996 !isa<ConstantSDNode>(Val: User->getOperand(Num: 1)) ||
3997 !isInt<12>(x: cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue()))
3998 return false;
3999
4000 FoundADDI = true;
4001
4002 // If we have a SHXADD instruction, prefer that over reassociating an ADDI.
4003 assert(Shift.getOpcode() == ISD::SHL);
4004 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
4005 if (Subtarget.hasShlAdd(ShAmt: ShiftAmt))
4006 return false;
4007
4008 // All users of the ADDI should be load/store.
4009 for (auto *ADDIUser : User->users())
4010 if (!isRegRegScaleLoadOrStore(User: ADDIUser, Add: SDValue(User, 0), Subtarget))
4011 return false;
4012 }
4013
4014 return true;
4015}
4016
4017bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
4018 ArrayRef<unsigned> Amounts,
4019 SDValue &Base, SDValue &Index,
4020 SDValue &Scale) {
4021 if (Addr.getOpcode() != ISD::ADD)
4022 return false;
4023 SDValue LHS = Addr.getOperand(i: 0);
4024 SDValue RHS = Addr.getOperand(i: 1);
4025
4026 EVT VT = Addr.getSimpleValueType();
4027 auto SelectShl = [this, VT, Amounts](SDValue N, SDValue &Index,
4028 SDValue &Shift) {
4029 if (N.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
4030 return false;
4031
4032 // Only match shifts by a value in range [0, MaxShiftAmount].
4033 unsigned ShiftAmt = N.getConstantOperandVal(i: 1);
4034 if (!llvm::is_contained(Range: Amounts, Element: ShiftAmt))
4035 return false;
4036
4037 Index = N.getOperand(i: 0);
4038 Shift = CurDAG->getTargetConstant(Val: ShiftAmt, DL: SDLoc(N), VT);
4039 return true;
4040 };
4041
4042 if (auto *C1 = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4043 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
4044 if (LHS.getOpcode() == ISD::ADD &&
4045 !isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4046 isInt<12>(x: C1->getSExtValue())) {
4047 if (SelectShl(LHS.getOperand(i: 1), Index, Scale) &&
4048 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 1))) {
4049 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
4050 DL: SDLoc(Addr), VT);
4051 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
4052 Op1: LHS.getOperand(i: 0), Op2: C1Val),
4053 0);
4054 return true;
4055 }
4056
4057 // Add is commutative so we need to check both operands.
4058 if (SelectShl(LHS.getOperand(i: 0), Index, Scale) &&
4059 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 0))) {
4060 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
4061 DL: SDLoc(Addr), VT);
4062 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
4063 Op1: LHS.getOperand(i: 1), Op2: C1Val),
4064 0);
4065 return true;
4066 }
4067 }
4068
4069 // Don't match add with constants.
4070 // FIXME: Is this profitable for large constants that have 0s in the lower
4071 // 12 bits that we can materialize with LUI?
4072 return false;
4073 }
4074
4075 // Try to match a shift on the RHS.
4076 if (SelectShl(RHS, Index, Scale)) {
4077 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: RHS))
4078 return false;
4079 Base = LHS;
4080 return true;
4081 }
4082
4083 // Try to match a shift on the LHS.
4084 if (SelectShl(LHS, Index, Scale)) {
4085 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: LHS))
4086 return false;
4087 Base = RHS;
4088 return true;
4089 }
4090
4091 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr))
4092 return false;
4093
4094 // Bail out if 0 is not in candidate shift amounts.
4095 if (!llvm::is_contained(Range&: Amounts, Element: 0))
4096 return false;
4097
4098 Base = LHS;
4099 Index = RHS;
4100 Scale = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT);
4101 return true;
4102}
4103
4104bool RISCVDAGToDAGISel::SelectAddrRegZextRegScale(SDValue Addr,
4105 ArrayRef<unsigned> Amounts,
4106 unsigned Bits, SDValue &Base,
4107 SDValue &Index,
4108 SDValue &Scale) {
4109 if (!SelectAddrRegRegScale(Addr, Amounts, Base, Index, Scale))
4110 return false;
4111
4112 if (Index.getOpcode() == ISD::AND) {
4113 auto *C = dyn_cast<ConstantSDNode>(Val: Index.getOperand(i: 1));
4114 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4115 Index = Index.getOperand(i: 0);
4116 return true;
4117 }
4118 }
4119
4120 return false;
4121}
4122
4123bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base,
4124 SDValue &Offset) {
4125 if (Addr.getOpcode() != ISD::ADD)
4126 return false;
4127
4128 if (isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)))
4129 return false;
4130
4131 Base = Addr.getOperand(i: 0);
4132 Offset = Addr.getOperand(i: 1);
4133 return true;
4134}
4135
4136bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
4137 SDValue &ShAmt) {
4138 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
4139 // amount. Look through an AND that doesn't affect those bits.
4140 if (!selectShiftAmountMask(N, ShiftWidth, ShAmt))
4141 ShAmt = N;
4142
4143 if (ShAmt.getOpcode() == ISD::ADD &&
4144 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4145 uint64_t Imm = ShAmt.getConstantOperandVal(i: 1);
4146 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
4147 // to avoid the ADD.
4148 if (Imm != 0 && Imm % ShiftWidth == 0) {
4149 ShAmt = ShAmt.getOperand(i: 0);
4150 return true;
4151 }
4152 } else if (ShAmt.getOpcode() == ISD::SUB &&
4153 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 0))) {
4154 uint64_t Imm = ShAmt.getConstantOperandVal(i: 0);
4155 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
4156 // generate a NEG instead of a SUB of a constant.
4157 if (Imm != 0 && Imm % ShiftWidth == 0) {
4158 SDLoc DL(ShAmt);
4159 EVT VT = ShAmt.getValueType();
4160 SDValue Zero = CurDAG->getRegister(Reg: RISCV::X0, VT);
4161 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
4162 MachineSDNode *Neg = CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero,
4163 Op2: ShAmt.getOperand(i: 1));
4164 ShAmt = SDValue(Neg, 0);
4165 return true;
4166 }
4167 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4168 // to generate a NOT instead of a SUB of a constant.
4169 if (Imm % ShiftWidth == ShiftWidth - 1) {
4170 SDLoc DL(ShAmt);
4171 EVT VT = ShAmt.getValueType();
4172 MachineSDNode *Not = CurDAG->getMachineNode(
4173 Opcode: RISCV::XORI, dl: DL, VT, Op1: ShAmt.getOperand(i: 1),
4174 Op2: CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/IsTarget: true));
4175 ShAmt = SDValue(Not, 0);
4176 return true;
4177 }
4178 }
4179
4180 return true;
4181}
4182
4183/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
4184/// check for equality with 0. This function emits instructions that convert the
4185/// seteq/setne into something that can be compared with 0.
4186/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
4187/// ISD::SETNE).
4188bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
4189 SDValue &Val, bool OneUse) {
4190 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
4191 "Unexpected condition code!");
4192
4193 // We're looking for a setcc.
4194 if (N->getOpcode() != ISD::SETCC)
4195 return false;
4196
4197 if (OneUse && !N->hasOneUse())
4198 return false;
4199
4200 // Must be an equality comparison.
4201 ISD::CondCode CCVal = N->getOperandCondCode(Num: 2);
4202 if (CCVal != ExpectedCCVal)
4203 return false;
4204
4205 SDValue LHS = N->getOperand(Num: 0);
4206 SDValue RHS = N->getOperand(Num: 1);
4207
4208 if (!LHS.getValueType().isScalarInteger())
4209 return false;
4210
4211 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
4212 if (isNullConstant(V: RHS)) {
4213 Val = LHS;
4214 return true;
4215 }
4216
4217 SDLoc DL(N);
4218
4219 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4220 int64_t CVal = C->getSExtValue();
4221 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
4222 // non-zero otherwise.
4223 if (CVal == -2048) {
4224 Val = SDValue(
4225 CurDAG->getMachineNode(
4226 Opcode: RISCV::XORI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4227 Op2: CurDAG->getSignedTargetConstant(Val: CVal, DL, VT: N->getValueType(ResNo: 0))),
4228 0);
4229 return true;
4230 }
4231 // If the RHS is [-2047,2048], we can use addi/addiw with -RHS to produce 0
4232 // if the LHS is equal to the RHS and non-zero otherwise.
4233 if (isInt<12>(x: CVal) || CVal == 2048) {
4234 unsigned Opc = RISCV::ADDI;
4235 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4236 cast<VTSDNode>(Val: LHS.getOperand(i: 1))->getVT() == MVT::i32) {
4237 Opc = RISCV::ADDIW;
4238 LHS = LHS.getOperand(i: 0);
4239 }
4240
4241 Val = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4242 Op2: CurDAG->getSignedTargetConstant(
4243 Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4244 0);
4245 return true;
4246 }
4247 if (isPowerOf2_64(Value: CVal) && Subtarget->hasStdExtZbs()) {
4248 Val = SDValue(
4249 CurDAG->getMachineNode(
4250 Opcode: RISCV::BINVI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4251 Op2: CurDAG->getTargetConstant(Val: Log2_64(Value: CVal), DL, VT: N->getValueType(ResNo: 0))),
4252 0);
4253 return true;
4254 }
4255 // Same as the addi case above but for larger immediates (signed 26-bit) use
4256 // the QC_E_ADDI instruction from the Xqcilia extension, if available. Avoid
4257 // anything which can be done with a single lui as it might be compressible.
4258 if (Subtarget->hasVendorXqcilia() && isInt<26>(x: CVal) &&
4259 (CVal & 0xFFF) != 0) {
4260 Val = SDValue(
4261 CurDAG->getMachineNode(
4262 Opcode: RISCV::QC_E_ADDI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4263 Op2: CurDAG->getSignedTargetConstant(Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4264 0);
4265 return true;
4266 }
4267 }
4268
4269 // If nothing else we can XOR the LHS and RHS to produce zero if they are
4270 // equal and a non-zero value if they aren't.
4271 Val = SDValue(
4272 CurDAG->getMachineNode(Opcode: RISCV::XOR, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS, Op2: RHS), 0);
4273 return true;
4274}
4275
4276bool RISCVDAGToDAGISel::selectSExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4277 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4278 cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT().getSizeInBits() == Bits) {
4279 Val = N.getOperand(i: 0);
4280 return true;
4281 }
4282
4283 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
4284 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
4285 return N;
4286
4287 SDValue N0 = N.getOperand(i: 0);
4288 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4289 N.getConstantOperandVal(i: 1) == ShiftAmt &&
4290 N0.getConstantOperandVal(i: 1) == ShiftAmt)
4291 return N0.getOperand(i: 0);
4292
4293 return N;
4294 };
4295
4296 MVT VT = N.getSimpleValueType();
4297 if (CurDAG->ComputeNumSignBits(Op: N) > (VT.getSizeInBits() - Bits)) {
4298 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
4299 return true;
4300 }
4301
4302 return false;
4303}
4304
4305bool RISCVDAGToDAGISel::selectZExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4306 if (N.getOpcode() == ISD::AND) {
4307 auto *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
4308 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4309 Val = N.getOperand(i: 0);
4310 return true;
4311 }
4312 }
4313 MVT VT = N.getSimpleValueType();
4314 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: Bits);
4315 if (CurDAG->MaskedValueIsZero(Op: N, Mask)) {
4316 Val = N;
4317 return true;
4318 }
4319
4320 return false;
4321}
4322
4323/// Look for various patterns that can be done with a SHL that can be folded
4324/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
4325/// SHXADD we are trying to match.
4326bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
4327 SDValue &Val) {
4328 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4329 MVT VT = N.getSimpleValueType();
4330 assert(VT == Subtarget->getXLenVT());
4331 SDValue N0 = N.getOperand(i: 0);
4332
4333 if (bool LeftShift = N0.getOpcode() == ISD::SHL;
4334 (LeftShift || N0.getOpcode() == ISD::SRL) &&
4335 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4336 uint64_t Mask = N.getConstantOperandVal(i: 1);
4337 unsigned C2 = N0.getConstantOperandVal(i: 1);
4338 assert(C2 > 0 && C2 < Subtarget->getXLen() && "Unexpected shift amount!");
4339
4340 unsigned XLen = Subtarget->getXLen();
4341 if (LeftShift)
4342 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4343 else
4344 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
4345
4346 if (isShiftedMask_64(Value: Mask)) {
4347 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4348 unsigned Trailing = llvm::countr_zero(Val: Mask);
4349 if (Trailing != ShAmt)
4350 return false;
4351
4352 unsigned Opcode;
4353 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
4354 // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
4355 // followed by a SHXADD with c3 for the X amount.
4356 if (LeftShift && Leading == 0 && C2 < Trailing)
4357 Opcode = RISCV::SRLI;
4358 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
4359 // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
4360 // followed by a SHXADD with c3 for the X amount.
4361 else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
4362 Opcode = RISCV::SRLIW;
4363 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
4364 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
4365 // followed by a SHXADD using c3 for the X amount.
4366 else if (!LeftShift && Leading == C2)
4367 Opcode = RISCV::SRLI;
4368 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
4369 // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
4370 // followed by a SHXADD using c3 for the X amount.
4371 else if (!LeftShift && Leading == 32 + C2)
4372 Opcode = RISCV::SRLIW;
4373 else
4374 return false;
4375
4376 SDLoc DL(N);
4377 ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
4378 Val = SDValue(
4379 CurDAG->getMachineNode(Opcode, dl: DL, VT, Op1: N0.getOperand(i: 0),
4380 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT)),
4381 0);
4382 return true;
4383 }
4384 } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
4385 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4386 uint64_t Mask = N.getConstantOperandVal(i: 1);
4387 unsigned C2 = N0.getConstantOperandVal(i: 1);
4388
4389 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
4390 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
4391 // use (srli (srai y, c2 - c3), c3 + c4) followed by a SHXADD with c4 as
4392 // the X amount.
4393 if (isShiftedMask_64(Value: Mask)) {
4394 unsigned XLen = Subtarget->getXLen();
4395 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4396 unsigned Trailing = llvm::countr_zero(Val: Mask);
4397 if (C2 > Leading && Leading > 0 && Trailing == ShAmt) {
4398 SDLoc DL(N);
4399 EVT VT = N.getValueType();
4400 Val = SDValue(CurDAG->getMachineNode(
4401 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4402 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT)),
4403 0);
4404 Val = SDValue(CurDAG->getMachineNode(
4405 Opcode: RISCV::SRLI, dl: DL, VT, Op1: Val,
4406 Op2: CurDAG->getTargetConstant(Val: Leading + ShAmt, DL, VT)),
4407 0);
4408 return true;
4409 }
4410 }
4411 }
4412 } else if (bool LeftShift = N.getOpcode() == ISD::SHL;
4413 (LeftShift || N.getOpcode() == ISD::SRL) &&
4414 isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4415 MVT VT = N.getSimpleValueType();
4416 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
4417 SDValue N0 = N.getOperand(i: 0);
4418 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
4419 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4420 uint64_t Mask = N0.getConstantOperandVal(i: 1);
4421 if (isShiftedMask_64(Value: Mask)) {
4422 unsigned C1 = N.getConstantOperandVal(i: 1);
4423 assert(C1 > 0 && C1 < Subtarget->getXLen() &&
4424 "Unexpected shift amount!");
4425 unsigned XLen = Subtarget->getXLen();
4426 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4427 unsigned Trailing = llvm::countr_zero(Val: Mask);
4428 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
4429 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
4430 if (LeftShift && Leading == 32 && Trailing > 0 &&
4431 (Trailing + C1) == ShAmt) {
4432 SDLoc DL(N);
4433 EVT VT = N.getValueType();
4434 Val = SDValue(CurDAG->getMachineNode(
4435 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4436 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4437 0);
4438 return true;
4439 }
4440 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
4441 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
4442 if (!LeftShift && Leading == 32 && Trailing > C1 &&
4443 (Trailing - C1) == ShAmt) {
4444 SDLoc DL(N);
4445 Val = SDValue(CurDAG->getMachineNode(
4446 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4447 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4448 0);
4449 return true;
4450 }
4451 }
4452 }
4453 }
4454
4455 return false;
4456}
4457
4458/// Look for various patterns that can be done with a SHL that can be folded
4459/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
4460/// SHXADD_UW we are trying to match.
4461bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
4462 SDValue &Val) {
4463 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
4464 N.hasOneUse()) {
4465 assert(N.getSimpleValueType() == Subtarget->getXLenVT() &&
4466 "Unexpected VT!");
4467 SDValue N0 = N.getOperand(i: 0);
4468 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4469 N0.hasOneUse()) {
4470 uint64_t Mask = N.getConstantOperandVal(i: 1);
4471 unsigned C2 = N0.getConstantOperandVal(i: 1);
4472 assert(C2 > 0 && C2 < Subtarget->getXLen() && "Unexpected shift amount!");
4473
4474 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4475
4476 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
4477 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
4478 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
4479 if (isShiftedMask_64(Value: Mask)) {
4480 unsigned Leading = llvm::countl_zero(Val: Mask);
4481 unsigned Trailing = llvm::countr_zero(Val: Mask);
4482 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
4483 SDLoc DL(N);
4484 EVT VT = N.getValueType();
4485 Val = SDValue(CurDAG->getMachineNode(
4486 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4487 Op2: CurDAG->getTargetConstant(Val: C2 - ShAmt, DL, VT)),
4488 0);
4489 return true;
4490 }
4491 }
4492 }
4493 }
4494
4495 return false;
4496}
4497
4498bool RISCVDAGToDAGISel::orDisjoint(const SDNode *N) const {
4499 assert(N->getOpcode() == ISD::OR || N->getOpcode() == RISCVISD::OR_VL);
4500 if (N->getFlags().hasDisjoint())
4501 return true;
4502 return CurDAG->haveNoCommonBitsSet(A: N->getOperand(Num: 0), B: N->getOperand(Num: 1));
4503}
4504
4505bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
4506 SDValue N, SDValue &Val) {
4507 int OrigCost = RISCVMatInt::getIntMatCost(Val: APInt(64, OrigImm), Size: 64, STI: *Subtarget,
4508 /*CompressionCost=*/true);
4509 int Cost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
4510 /*CompressionCost=*/true);
4511 if (OrigCost <= Cost)
4512 return false;
4513
4514 Val = selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm, Subtarget: *Subtarget);
4515 return true;
4516}
4517
4518bool RISCVDAGToDAGISel::selectZExtImm32(SDValue N, SDValue &Val) {
4519 if (!isa<ConstantSDNode>(Val: N))
4520 return false;
4521 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4522 if ((Imm >> 31) != 1)
4523 return false;
4524
4525 for (const SDNode *U : N->users()) {
4526 switch (U->getOpcode()) {
4527 case ISD::ADD:
4528 break;
4529 case ISD::OR:
4530 if (orDisjoint(N: U))
4531 break;
4532 return false;
4533 default:
4534 return false;
4535 }
4536 }
4537
4538 return selectImm64IfCheaper(Imm: 0xffffffff00000000 | Imm, OrigImm: Imm, N, Val);
4539}
4540
4541bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
4542 if (!isa<ConstantSDNode>(Val: N))
4543 return false;
4544 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4545 if (isInt<32>(x: Imm))
4546 return false;
4547 if (Imm == INT64_MIN)
4548 return false;
4549
4550 for (const SDNode *U : N->users()) {
4551 switch (U->getOpcode()) {
4552 case ISD::ADD:
4553 break;
4554 case RISCVISD::VMV_V_X_VL:
4555 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4556 return V->getOpcode() == ISD::ADD ||
4557 V->getOpcode() == RISCVISD::ADD_VL;
4558 }))
4559 return false;
4560 break;
4561 default:
4562 return false;
4563 }
4564 }
4565
4566 return selectImm64IfCheaper(Imm: -Imm, OrigImm: Imm, N, Val);
4567}
4568
4569bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
4570 if (!isa<ConstantSDNode>(Val: N))
4571 return false;
4572 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4573
4574 // For 32-bit signed constants, we can only substitute LUI+ADDI with LUI.
4575 if (isInt<32>(x: Imm) && ((Imm & 0xfff) != 0xfff || Imm == -1))
4576 return false;
4577
4578 // Abandon this transform if the constant is needed elsewhere.
4579 for (const SDNode *U : N->users()) {
4580 switch (U->getOpcode()) {
4581 case ISD::AND:
4582 case ISD::OR:
4583 case ISD::XOR:
4584 if (!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()))
4585 return false;
4586 break;
4587 case RISCVISD::VMV_V_X_VL:
4588 if (!Subtarget->hasStdExtZvkb())
4589 return false;
4590 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4591 return V->getOpcode() == ISD::AND ||
4592 V->getOpcode() == RISCVISD::AND_VL;
4593 }))
4594 return false;
4595 break;
4596 default:
4597 return false;
4598 }
4599 }
4600
4601 if (isInt<32>(x: Imm)) {
4602 Val =
4603 selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm: ~Imm, Subtarget: *Subtarget);
4604 return true;
4605 }
4606
4607 // For 64-bit constants, the instruction sequences get complex,
4608 // so we select inverted only if it's cheaper.
4609 return selectImm64IfCheaper(Imm: ~Imm, OrigImm: Imm, N, Val);
4610}
4611
4612static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
4613 unsigned Bits,
4614 const TargetInstrInfo *TII) {
4615 unsigned MCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: User->getMachineOpcode());
4616
4617 if (!MCOpcode)
4618 return false;
4619
4620 const MCInstrDesc &MCID = TII->get(Opcode: User->getMachineOpcode());
4621 const uint64_t TSFlags = MCID.TSFlags;
4622 if (!RISCVII::hasSEWOp(TSFlags))
4623 return false;
4624 assert(RISCVII::hasVLOp(TSFlags));
4625
4626 unsigned ChainOpIdx = User->getNumOperands() - 1;
4627 bool HasChainOp = User->getOperand(Num: ChainOpIdx).getValueType() == MVT::Other;
4628 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
4629 unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
4630 const unsigned Log2SEW = User->getConstantOperandVal(Num: VLIdx + 1);
4631
4632 if (UserOpNo == VLIdx)
4633 return false;
4634
4635 auto NumDemandedBits =
4636 RISCV::getVectorLowDemandedScalarBits(Opcode: MCOpcode, Log2SEW);
4637 return NumDemandedBits && Bits >= *NumDemandedBits;
4638}
4639
4640// Return true if all users of this SDNode* only consume the lower \p Bits.
4641// This can be used to form W instructions for add/sub/mul/shl even when the
4642// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
4643// SimplifyDemandedBits has made it so some users see a sext_inreg and some
4644// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
4645// the add/sub/mul/shl to become non-W instructions. By checking the users we
4646// may be able to use a W instruction and CSE with the other instruction if
4647// this has happened. We could try to detect that the CSE opportunity exists
4648// before doing this, but that would be more complicated.
4649bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits,
4650 const unsigned Depth) const {
4651 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
4652 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
4653 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
4654 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
4655 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
4656 isa<ConstantSDNode>(Node) || Depth != 0) &&
4657 "Unexpected opcode");
4658
4659 if (Depth >= SelectionDAG::MaxRecursionDepth)
4660 return false;
4661
4662 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
4663 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
4664 if (Depth == 0 && !Node->getValueType(ResNo: 0).isScalarInteger())
4665 return false;
4666
4667 for (SDUse &Use : Node->uses()) {
4668 SDNode *User = Use.getUser();
4669 // Users of this node should have already been instruction selected
4670 if (!User->isMachineOpcode())
4671 return false;
4672
4673 // TODO: Add more opcodes?
4674 switch (User->getMachineOpcode()) {
4675 default:
4676 if (vectorPseudoHasAllNBitUsers(User, UserOpNo: Use.getOperandNo(), Bits, TII))
4677 break;
4678 return false;
4679 case RISCV::ADDW:
4680 case RISCV::ADDIW:
4681 case RISCV::SUBW:
4682 case RISCV::MULW:
4683 case RISCV::SLLW:
4684 case RISCV::SLLIW:
4685 case RISCV::SRAW:
4686 case RISCV::SRAIW:
4687 case RISCV::SRLW:
4688 case RISCV::SRLIW:
4689 case RISCV::DIVW:
4690 case RISCV::DIVUW:
4691 case RISCV::REMW:
4692 case RISCV::REMUW:
4693 case RISCV::ROLW:
4694 case RISCV::RORW:
4695 case RISCV::RORIW:
4696 case RISCV::CLSW:
4697 case RISCV::CLZW:
4698 case RISCV::CTZW:
4699 case RISCV::CPOPW:
4700 case RISCV::SLLI_UW:
4701 case RISCV::ABSW:
4702 case RISCV::FMV_W_X:
4703 case RISCV::FCVT_H_W:
4704 case RISCV::FCVT_H_W_INX:
4705 case RISCV::FCVT_H_WU:
4706 case RISCV::FCVT_H_WU_INX:
4707 case RISCV::FCVT_S_W:
4708 case RISCV::FCVT_S_W_INX:
4709 case RISCV::FCVT_S_WU:
4710 case RISCV::FCVT_S_WU_INX:
4711 case RISCV::FCVT_D_W:
4712 case RISCV::FCVT_D_W_INX:
4713 case RISCV::FCVT_D_WU:
4714 case RISCV::FCVT_D_WU_INX:
4715 case RISCV::TH_REVW:
4716 case RISCV::TH_SRRIW:
4717 if (Bits >= 32)
4718 break;
4719 return false;
4720 case RISCV::SLL:
4721 case RISCV::SRA:
4722 case RISCV::SRL:
4723 case RISCV::ROL:
4724 case RISCV::ROR:
4725 case RISCV::BSET:
4726 case RISCV::BCLR:
4727 case RISCV::BINV:
4728 case RISCV::BEXT:
4729 // Shift amount and bit index operands only use log2(Xlen) bits.
4730 if (Use.getOperandNo() == 1 && Bits >= Log2_32(Value: Subtarget->getXLen()))
4731 break;
4732 return false;
4733 case RISCV::SLLI:
4734 // SLLI only uses the lower (XLen - ShAmt) bits.
4735 if (Bits >= Subtarget->getXLen() - User->getConstantOperandVal(Num: 1))
4736 break;
4737 return false;
4738 case RISCV::ANDI:
4739 if (Bits >= (unsigned)llvm::bit_width(Value: User->getConstantOperandVal(Num: 1)))
4740 break;
4741 goto RecCheck;
4742 case RISCV::ORI: {
4743 uint64_t Imm = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
4744 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(Value: ~Imm))
4745 break;
4746 [[fallthrough]];
4747 }
4748 case RISCV::AND:
4749 case RISCV::OR:
4750 case RISCV::XOR:
4751 case RISCV::XORI:
4752 case RISCV::ANDN:
4753 case RISCV::ORN:
4754 case RISCV::XNOR:
4755 case RISCV::SH1ADD:
4756 case RISCV::SH2ADD:
4757 case RISCV::SH3ADD:
4758 case RISCV::PseudoMERGE:
4759 RecCheck:
4760 if (hasAllNBitUsers(Node: User, Bits, Depth: Depth + 1))
4761 break;
4762 return false;
4763 case RISCV::SRLI: {
4764 unsigned ShAmt = User->getConstantOperandVal(Num: 1);
4765 // If we are shifting right by less than Bits, and users don't demand any
4766 // bits that were shifted into [Bits-1:0], then we can consider this as an
4767 // N-Bit user.
4768 if (Bits > ShAmt && hasAllNBitUsers(Node: User, Bits: Bits - ShAmt, Depth: Depth + 1))
4769 break;
4770 return false;
4771 }
4772 case RISCV::SEXT_B:
4773 case RISCV::PACKH:
4774 if (Bits >= 8)
4775 break;
4776 return false;
4777 case RISCV::SEXT_H:
4778 case RISCV::FMV_H_X:
4779 case RISCV::ZEXT_H_RV32:
4780 case RISCV::ZEXT_H_RV64:
4781 case RISCV::PACKW:
4782 if (Bits >= 16)
4783 break;
4784 return false;
4785 case RISCV::PACK:
4786 if (Bits >= (Subtarget->getXLen() / 2))
4787 break;
4788 return false;
4789 case RISCV::PPAIRE_H:
4790 // If only the lower 32-bits of the result are used, then only the
4791 // lower 16 bits of the inputs are used.
4792 if (Bits >= 16 && hasAllNBitUsers(Node: User, Bits: 32, Depth: Depth + 1))
4793 break;
4794 return false;
4795 case RISCV::ADD_UW:
4796 case RISCV::SH1ADD_UW:
4797 case RISCV::SH2ADD_UW:
4798 case RISCV::SH3ADD_UW:
4799 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
4800 // 32 bits.
4801 if (Use.getOperandNo() == 0 && Bits >= 32)
4802 break;
4803 return false;
4804 case RISCV::SB:
4805 if (Use.getOperandNo() == 0 && Bits >= 8)
4806 break;
4807 return false;
4808 case RISCV::SH:
4809 if (Use.getOperandNo() == 0 && Bits >= 16)
4810 break;
4811 return false;
4812 case RISCV::SW:
4813 if (Use.getOperandNo() == 0 && Bits >= 32)
4814 break;
4815 return false;
4816 case RISCV::TH_EXT:
4817 case RISCV::TH_EXTU: {
4818 unsigned Msb = User->getConstantOperandVal(Num: 1);
4819 unsigned Lsb = User->getConstantOperandVal(Num: 2);
4820 // Behavior of Msb < Lsb is not well documented.
4821 if (Msb >= Lsb && Bits > Msb)
4822 break;
4823 return false;
4824 }
4825 }
4826 }
4827
4828 return true;
4829}
4830
4831// Select a constant that can be represented as (sign_extend(imm5) << imm2).
4832bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
4833 SDValue &Shl2) {
4834 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4835 if (!C)
4836 return false;
4837
4838 int64_t Offset = C->getSExtValue();
4839 for (unsigned Shift = 0; Shift < 4; Shift++) {
4840 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0)) {
4841 EVT VT = N->getValueType(ResNo: 0);
4842 Simm5 = CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(N), VT);
4843 Shl2 = CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(N), VT);
4844 return true;
4845 }
4846 }
4847
4848 return false;
4849}
4850
4851// Select VL as a 5 bit immediate or a value that will become a register. This
4852// allows us to choose between VSETIVLI or VSETVLI later.
4853bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
4854 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4855 if (C && isUInt<5>(x: C->getZExtValue())) {
4856 VL = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(N),
4857 VT: N->getValueType(ResNo: 0));
4858 } else if (C && C->isAllOnes()) {
4859 // Treat all ones as VLMax.
4860 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4861 VT: N->getValueType(ResNo: 0));
4862 } else if (isa<RegisterSDNode>(Val: N) &&
4863 cast<RegisterSDNode>(Val&: N)->getReg() == RISCV::X0) {
4864 // All our VL operands use an operand that allows GPRNoX0 or an immediate
4865 // as the register class. Convert X0 to a special immediate to pass the
4866 // MachineVerifier. This is recognized specially by the vsetvli insertion
4867 // pass.
4868 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4869 VT: N->getValueType(ResNo: 0));
4870 } else {
4871 VL = N;
4872 }
4873
4874 return true;
4875}
4876
4877static SDValue findVSplat(SDValue N) {
4878 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
4879 if (!N.getOperand(i: 0).isUndef())
4880 return SDValue();
4881 N = N.getOperand(i: 1);
4882 }
4883 SDValue Splat = N;
4884 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
4885 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
4886 !Splat.getOperand(i: 0).isUndef())
4887 return SDValue();
4888 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
4889 return Splat;
4890}
4891
4892bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
4893 SDValue Splat = findVSplat(N);
4894 if (!Splat)
4895 return false;
4896
4897 SplatVal = Splat.getOperand(i: 1);
4898 return true;
4899}
4900
4901static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
4902 SelectionDAG &DAG,
4903 const RISCVSubtarget &Subtarget,
4904 std::function<bool(int64_t)> ValidateImm,
4905 bool Decrement = false) {
4906 SDValue Splat = findVSplat(N);
4907 if (!Splat || !isa<ConstantSDNode>(Val: Splat.getOperand(i: 1)))
4908 return false;
4909
4910 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
4911 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
4912 "Unexpected splat operand type");
4913
4914 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
4915 // type is wider than the resulting vector element type: an implicit
4916 // truncation first takes place. Therefore, perform a manual
4917 // truncation/sign-extension in order to ignore any truncated bits and catch
4918 // any zero-extended immediate.
4919 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
4920 // sign-extending to (XLenVT -1).
4921 APInt SplatConst = Splat.getConstantOperandAPInt(i: 1).sextOrTrunc(width: SplatEltSize);
4922
4923 int64_t SplatImm = SplatConst.getSExtValue();
4924
4925 if (!ValidateImm(SplatImm))
4926 return false;
4927
4928 if (Decrement)
4929 SplatImm -= 1;
4930
4931 SplatVal =
4932 DAG.getSignedTargetConstant(Val: SplatImm, DL: SDLoc(N), VT: Subtarget.getXLenVT());
4933 return true;
4934}
4935
4936bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
4937 return selectVSplatImmHelper(N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4938 ValidateImm: [](int64_t Imm) { return isInt<5>(x: Imm); });
4939}
4940
4941bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
4942 return selectVSplatImmHelper(
4943 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4944 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4945 /*Decrement=*/true);
4946}
4947
4948bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal) {
4949 return selectVSplatImmHelper(
4950 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4951 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4952 /*Decrement=*/false);
4953}
4954
4955bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
4956 SDValue &SplatVal) {
4957 return selectVSplatImmHelper(
4958 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4959 ValidateImm: [](int64_t Imm) { return Imm != 0 && Imm >= -15 && Imm <= 16; },
4960 /*Decrement=*/true);
4961}
4962
4963bool RISCVDAGToDAGISel::selectVSplatUimm(SDValue N, unsigned Bits,
4964 SDValue &SplatVal) {
4965 return selectVSplatImmHelper(
4966 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4967 ValidateImm: [Bits](int64_t Imm) { return isUIntN(N: Bits, x: Imm); });
4968}
4969
4970bool RISCVDAGToDAGISel::selectVSplatImm64Neg(SDValue N, SDValue &SplatVal) {
4971 SDValue Splat = findVSplat(N);
4972 return Splat && selectNegImm(N: Splat.getOperand(i: 1), Val&: SplatVal);
4973}
4974
4975bool RISCVDAGToDAGISel::selectLow8BitsVSplat(SDValue N, SDValue &SplatVal) {
4976 auto IsExtOrTrunc = [](SDValue N) {
4977 switch (N->getOpcode()) {
4978 case ISD::SIGN_EXTEND:
4979 case ISD::ZERO_EXTEND:
4980 // There's no passthru on these _VL nodes so any VL/mask is ok, since any
4981 // inactive elements will be undef.
4982 case RISCVISD::TRUNCATE_VECTOR_VL:
4983 case RISCVISD::VSEXT_VL:
4984 case RISCVISD::VZEXT_VL:
4985 return true;
4986 default:
4987 return false;
4988 }
4989 };
4990
4991 // We can have multiple nested nodes, so unravel them all if needed.
4992 while (IsExtOrTrunc(N)) {
4993 if (!N.hasOneUse() || N.getScalarValueSizeInBits() < 8)
4994 return false;
4995 N = N->getOperand(Num: 0);
4996 }
4997
4998 return selectVSplat(N, SplatVal);
4999}
5000
5001bool RISCVDAGToDAGISel::selectScalarFPAsInt(SDValue N, SDValue &Imm) {
5002 // Allow bitcasts from XLenVT -> FP.
5003 if (N.getOpcode() == ISD::BITCAST &&
5004 N.getOperand(i: 0).getValueType() == Subtarget->getXLenVT()) {
5005 Imm = N.getOperand(i: 0);
5006 return true;
5007 }
5008 // Allow moves from XLenVT to FP.
5009 if (N.getOpcode() == RISCVISD::FMV_H_X ||
5010 N.getOpcode() == RISCVISD::FMV_W_X_RV64) {
5011 Imm = N.getOperand(i: 0);
5012 return true;
5013 }
5014
5015 // Otherwise, look for FP constants that can materialized with scalar int.
5016 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val: N.getNode());
5017 if (!CFP)
5018 return false;
5019 const APFloat &APF = CFP->getValueAPF();
5020 // td can handle +0.0 already.
5021 if (APF.isPosZero())
5022 return false;
5023
5024 MVT VT = CFP->getSimpleValueType(ResNo: 0);
5025
5026 MVT XLenVT = Subtarget->getXLenVT();
5027 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
5028 assert(APF.isNegZero() && "Unexpected constant.");
5029 return false;
5030 }
5031 SDLoc DL(N);
5032 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
5033 Subtarget: *Subtarget);
5034 return true;
5035}
5036
5037bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
5038 SDValue &Imm) {
5039 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N)) {
5040 int64_t ImmVal = SignExtend64(X: C->getSExtValue(), B: Width);
5041
5042 if (!isInt<5>(x: ImmVal))
5043 return false;
5044
5045 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N),
5046 VT: Subtarget->getXLenVT());
5047 return true;
5048 }
5049
5050 return false;
5051}
5052
5053// Match XOR with a VMSET_VL operand. Return the other operand.
5054bool RISCVDAGToDAGISel::selectVMNOTOp(SDValue N, SDValue &Res) {
5055 if (N.getOpcode() != ISD::XOR)
5056 return false;
5057
5058 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5059 Res = N.getOperand(i: 1);
5060 return true;
5061 }
5062
5063 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5064 Res = N.getOperand(i: 0);
5065 return true;
5066 }
5067
5068 return false;
5069}
5070
5071// Match VMXOR_VL with a VMSET_VL operand. Making sure that that VL operand
5072// matches the parent's VL. Return the other operand of the VMXOR_VL.
5073bool RISCVDAGToDAGISel::selectVMNOT_VLOp(SDNode *Parent, SDValue N,
5074 SDValue &Res) {
5075 if (N.getOpcode() != RISCVISD::VMXOR_VL)
5076 return false;
5077
5078 assert(Parent &&
5079 (Parent->getOpcode() == RISCVISD::VMAND_VL ||
5080 Parent->getOpcode() == RISCVISD::VMOR_VL ||
5081 Parent->getOpcode() == RISCVISD::VMXOR_VL) &&
5082 "Unexpected parent");
5083
5084 // The VL should match the parent.
5085 if (Parent->getOperand(Num: 2) != N->getOperand(Num: 2))
5086 return false;
5087
5088 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5089 Res = N.getOperand(i: 1);
5090 return true;
5091 }
5092
5093 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5094 Res = N.getOperand(i: 0);
5095 return true;
5096 }
5097
5098 return false;
5099}
5100
5101// Try to remove sext.w if the input is a W instruction or can be made into
5102// a W instruction cheaply.
5103bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
5104 // Look for the sext.w pattern, addiw rd, rs1, 0.
5105 if (N->getMachineOpcode() != RISCV::ADDIW ||
5106 !isNullConstant(V: N->getOperand(Num: 1)))
5107 return false;
5108
5109 SDValue N0 = N->getOperand(Num: 0);
5110 if (!N0.isMachineOpcode())
5111 return false;
5112
5113 switch (N0.getMachineOpcode()) {
5114 default:
5115 break;
5116 case RISCV::ADD:
5117 case RISCV::ADDI:
5118 case RISCV::SUB:
5119 case RISCV::MUL:
5120 case RISCV::SLLI: {
5121 // Convert sext.w+add/sub/mul to their W instructions. This will create
5122 // a new independent instruction. This improves latency.
5123 unsigned Opc;
5124 switch (N0.getMachineOpcode()) {
5125 default:
5126 llvm_unreachable("Unexpected opcode!");
5127 case RISCV::ADD: Opc = RISCV::ADDW; break;
5128 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
5129 case RISCV::SUB: Opc = RISCV::SUBW; break;
5130 case RISCV::MUL: Opc = RISCV::MULW; break;
5131 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
5132 }
5133
5134 SDValue N00 = N0.getOperand(i: 0);
5135 SDValue N01 = N0.getOperand(i: 1);
5136
5137 // Shift amount needs to be uimm5.
5138 if (N0.getMachineOpcode() == RISCV::SLLI &&
5139 !isUInt<5>(x: cast<ConstantSDNode>(Val&: N01)->getSExtValue()))
5140 break;
5141
5142 SDNode *Result =
5143 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VT: N->getValueType(ResNo: 0),
5144 Op1: N00, Op2: N01);
5145 ReplaceUses(F: N, T: Result);
5146 return true;
5147 }
5148 case RISCV::ADDW:
5149 case RISCV::ADDIW:
5150 case RISCV::SUBW:
5151 case RISCV::MULW:
5152 case RISCV::SLLIW:
5153 case RISCV::PACKW:
5154 case RISCV::TH_MULAW:
5155 case RISCV::TH_MULAH:
5156 case RISCV::TH_MULSW:
5157 case RISCV::TH_MULSH:
5158 if (N0.getValueType() == MVT::i32)
5159 break;
5160
5161 // Result is already sign extended just remove the sext.w.
5162 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
5163 ReplaceUses(F: N, T: N0.getNode());
5164 return true;
5165 }
5166
5167 return false;
5168}
5169
5170static bool usesAllOnesMask(SDValue MaskOp) {
5171 const auto IsVMSet = [](unsigned Opc) {
5172 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
5173 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
5174 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
5175 Opc == RISCV::PseudoVMSET_M_B8;
5176 };
5177
5178 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
5179 // undefined behaviour if it's the wrong bitwidth, so we could choose to
5180 // assume that it's all-ones? Same applies to its VL.
5181 return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode());
5182}
5183
5184static bool isImplicitDef(SDValue V) {
5185 if (!V.isMachineOpcode())
5186 return false;
5187 if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
5188 for (unsigned I = 1; I < V.getNumOperands(); I += 2)
5189 if (!isImplicitDef(V: V.getOperand(i: I)))
5190 return false;
5191 return true;
5192 }
5193 return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
5194}
5195
5196// Optimize masked RVV pseudo instructions with a known all-ones mask to their
5197// corresponding "unmasked" pseudo versions.
5198bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
5199 const RISCV::RISCVMaskedPseudoInfo *I =
5200 RISCV::getMaskedPseudoInfo(MaskedPseudo: N->getMachineOpcode());
5201 if (!I)
5202 return false;
5203
5204 unsigned MaskOpIdx = I->MaskOpIdx;
5205 if (!usesAllOnesMask(MaskOp: N->getOperand(Num: MaskOpIdx)))
5206 return false;
5207
5208 // There are two classes of pseudos in the table - compares and
5209 // everything else. See the comment on RISCVMaskedPseudo for details.
5210 const unsigned Opc = I->UnmaskedPseudo;
5211 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
5212 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MCID);
5213
5214 const MCInstrDesc &MaskedMCID = TII->get(Opcode: N->getMachineOpcode());
5215 const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MaskedMCID);
5216
5217 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
5218 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&
5219 "Unmasked pseudo has policy but masked pseudo doesn't?");
5220 assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru &&
5221 "Unexpected pseudo structure");
5222 assert(!(HasPassthru && !MaskedHasPassthru) &&
5223 "Unmasked pseudo has passthru but masked pseudo doesn't?");
5224
5225 SmallVector<SDValue, 8> Ops;
5226 // Skip the passthru operand at index 0 if the unmasked don't have one.
5227 bool ShouldSkip = !HasPassthru && MaskedHasPassthru;
5228 bool DropPolicy = !RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags) &&
5229 RISCVII::hasVecPolicyOp(TSFlags: MaskedMCID.TSFlags);
5230 bool HasChainOp =
5231 N->getOperand(Num: N->getNumOperands() - 1).getValueType() == MVT::Other;
5232 unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp;
5233 for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) {
5234 // Skip the mask
5235 SDValue Op = N->getOperand(Num: I);
5236 if (I == MaskOpIdx)
5237 continue;
5238 if (DropPolicy && I == LastOpNum)
5239 continue;
5240 Ops.push_back(Elt: Op);
5241 }
5242
5243 MachineSDNode *Result =
5244 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5245
5246 if (!N->memoperands_empty())
5247 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: N->memoperands());
5248
5249 Result->setFlags(N->getFlags());
5250 ReplaceUses(F: N, T: Result);
5251
5252 return true;
5253}
5254
5255/// If our passthru is an implicit_def, use noreg instead. This side
5256/// steps issues with MachineCSE not being able to CSE expressions with
5257/// IMPLICIT_DEF operands while preserving the semantic intent. See
5258/// pr64282 for context. Note that this transform is the last one
5259/// performed at ISEL DAG to DAG.
5260bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
5261 bool MadeChange = false;
5262 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5263
5264 while (Position != CurDAG->allnodes_begin()) {
5265 SDNode *N = &*--Position;
5266 if (N->use_empty() || !N->isMachineOpcode())
5267 continue;
5268
5269 const unsigned Opc = N->getMachineOpcode();
5270 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opc) ||
5271 !RISCVII::isFirstDefTiedToFirstUse(Desc: TII->get(Opcode: Opc)) ||
5272 !isImplicitDef(V: N->getOperand(Num: 0)))
5273 continue;
5274
5275 SmallVector<SDValue> Ops;
5276 Ops.push_back(Elt: CurDAG->getRegister(Reg: RISCV::NoRegister, VT: N->getValueType(ResNo: 0)));
5277 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
5278 SDValue Op = N->getOperand(Num: I);
5279 Ops.push_back(Elt: Op);
5280 }
5281
5282 MachineSDNode *Result =
5283 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5284 Result->setFlags(N->getFlags());
5285 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: cast<MachineSDNode>(Val: N)->memoperands());
5286 ReplaceUses(F: N, T: Result);
5287 MadeChange = true;
5288 }
5289 return MadeChange;
5290}
5291
5292
5293// This pass converts a legalized DAG into a RISCV-specific DAG, ready
5294// for instruction scheduling.
5295FunctionPass *llvm::createRISCVISelDagLegacyPass(RISCVTargetMachine &TM,
5296 CodeGenOptLevel OptLevel) {
5297 return new RISCVDAGToDAGISelLegacy(TM, OptLevel);
5298}
5299
5300RISCVISelDAGToDAGPass::RISCVISelDAGToDAGPass(RISCVTargetMachine &TM,
5301 CodeGenOptLevel OptLevel)
5302 : SelectionDAGISelPass(std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5303
5304char RISCVDAGToDAGISelLegacy::ID = 0;
5305
5306RISCVDAGToDAGISelLegacy::RISCVDAGToDAGISelLegacy(RISCVTargetMachine &TM,
5307 CodeGenOptLevel OptLevel)
5308 : SelectionDAGISelLegacy(
5309 ID, std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5310
5311INITIALIZE_PASS(RISCVDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
5312