1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "MCTargetDesc/RISCVMatInt.h"
17#include "RISCVISelLowering.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSelectionDAGInfo.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/Support/Alignment.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
32#define GET_DAGISEL_BODY RISCVDAGToDAGISel
33#include "RISCVGenDAGISel.inc"
34
35void RISCVDAGToDAGISel::PreprocessISelDAG() {
36 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
37
38 bool MadeChange = false;
39 while (Position != CurDAG->allnodes_begin()) {
40 SDNode *N = &*--Position;
41 if (N->use_empty())
42 continue;
43
44 SDValue Result;
45 switch (N->getOpcode()) {
46 case ISD::SPLAT_VECTOR: {
47 if (Subtarget->hasStdExtP())
48 break;
49 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
50 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
51 MVT VT = N->getSimpleValueType(ResNo: 0);
52 unsigned Opc =
53 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
54 SDLoc DL(N);
55 SDValue VL = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
56 SDValue Src = N->getOperand(Num: 0);
57 if (VT.isInteger())
58 Src = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget->getXLenVT(),
59 Operand: N->getOperand(Num: 0));
60 Result = CurDAG->getNode(Opcode: Opc, DL, VT, N1: CurDAG->getUNDEF(VT), N2: Src, N3: VL);
61 break;
62 }
63 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
64 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
65 // load. Done after lowering and combining so that we have a chance to
66 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
67 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
68 MVT VT = N->getSimpleValueType(ResNo: 0);
69 SDValue Passthru = N->getOperand(Num: 0);
70 SDValue Lo = N->getOperand(Num: 1);
71 SDValue Hi = N->getOperand(Num: 2);
72 SDValue VL = N->getOperand(Num: 3);
73 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
74 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
75 "Unexpected VTs!");
76 MachineFunction &MF = CurDAG->getMachineFunction();
77 SDLoc DL(N);
78
79 // Create temporary stack for each expanding node.
80 SDValue StackSlot =
81 CurDAG->CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 8), Alignment: Align(8));
82 int FI = cast<FrameIndexSDNode>(Val: StackSlot.getNode())->getIndex();
83 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
84
85 SDValue Chain = CurDAG->getEntryNode();
86 Lo = CurDAG->getStore(Chain, dl: DL, Val: Lo, Ptr: StackSlot, PtrInfo: MPI, Alignment: Align(8));
87
88 SDValue OffsetSlot =
89 CurDAG->getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL);
90 Hi = CurDAG->getStore(Chain, dl: DL, Val: Hi, Ptr: OffsetSlot, PtrInfo: MPI.getWithOffset(O: 4),
91 Alignment: Align(8));
92
93 Chain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
94
95 SDVTList VTs = CurDAG->getVTList(VTs: {VT, MVT::Other});
96 SDValue IntID =
97 CurDAG->getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: MVT::i64);
98 SDValue Ops[] = {Chain,
99 IntID,
100 Passthru,
101 StackSlot,
102 CurDAG->getRegister(Reg: RISCV::X0, VT: MVT::i64),
103 VL};
104
105 Result = CurDAG->getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
106 MemVT: MVT::i64, PtrInfo: MPI, Alignment: Align(8),
107 Flags: MachineMemOperand::MOLoad);
108 break;
109 }
110 case ISD::FP_EXTEND: {
111 // We only have vector patterns for riscv_fpextend_vl in isel.
112 SDLoc DL(N);
113 MVT VT = N->getSimpleValueType(ResNo: 0);
114 if (!VT.isVector())
115 break;
116 SDValue VLMAX = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
117 SDValue TrueMask = CurDAG->getNode(
118 Opcode: RISCVISD::VMSET_VL, DL, VT: VT.changeVectorElementType(EltVT: MVT::i1), Operand: VLMAX);
119 Result = CurDAG->getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT, N1: N->getOperand(Num: 0),
120 N2: TrueMask, N3: VLMAX);
121 break;
122 }
123 case ISD::ADD: {
124 // Turn (add X, C) into (sub X, -C) when a constant node holding -C
125 // already exists in the DAG, so both share one materialization. Do this
126 // before selection, while both are still ConstantSDNodes: by selection
127 // time -C may already have been selected into instructions.
128 //
129 // ADD is commutative, but getNode canonicalizes constants to the RHS, so
130 // the constant is always operand 1.
131 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
132 if (!N1C)
133 break;
134 MVT VT = N->getSimpleValueType(ResNo: 0);
135 if (VT != Subtarget->getXLenVT())
136 break;
137 int64_t Imm = N1C->getSExtValue();
138 // Only worthwhile for wide constants: values that fit in 32 bits take at
139 // most two instructions to materialize, matching the threshold used by
140 // selectNegImm. Skip INT64_MIN too, whose negation is itself.
141 if (isInt<32>(x: Imm) || Imm == INT64_MIN)
142 break;
143 // A constant is anchored if it has a user other than an ADD, i.e. it is
144 // materialized regardless of this fold. N1C is the (unique) node for Imm,
145 // so the positive side needs no search.
146 auto IsAnchored = [](const SDNode *C) {
147 return any_of(Range: C->users(), P: [](const SDNode *U) {
148 return U->getOpcode() != ISD::ADD;
149 });
150 };
151 // If Imm is materialized anyway, keep the ADD so it reuses Imm; an ADD is
152 // also more compressible than a SUB. This also lets us skip the search
153 // for -Imm below.
154 if (IsAnchored(N1C))
155 break;
156 // Find the (unique) constant node for -Imm, if any.
157 const SDNode *NegC = nullptr;
158 for (const SDNode &Node : CurDAG->allnodes()) {
159 auto *C = dyn_cast<ConstantSDNode>(Val: &Node);
160 if (C && C->getSimpleValueType(ResNo: 0) == VT && C->getSExtValue() == -Imm) {
161 NegC = &Node;
162 break;
163 }
164 }
165 // Reuse is only free if -Imm is already in the DAG.
166 if (!NegC)
167 break;
168 // dyn_cast<ConstantSDNode> also matches TargetConstant, which is encoded
169 // into the instruction rather than materialized, so reusing it would not
170 // remove a materialization. No TargetConstant is this wide (the largest
171 // are intrinsic IDs, which fit in 32 bits), so assert it is a Constant.
172 assert(NegC->getOpcode() == ISD::Constant &&
173 "Unexpected wide TargetConstant");
174 // Pick which of Imm/-Imm should be the surviving constant, so exactly
175 // one of the pair is materialized and any ADDs of the other reuse it:
176 // - if -Imm is materialized anyway, reuse it (rewrite to SUB);
177 // - else keep the cheaper constant, breaking ties towards the positive
178 // value so both ADDs of a C/-C pair agree on the survivor.
179 bool Rewrite;
180 if (IsAnchored(NegC)) {
181 Rewrite = true;
182 } else {
183 int PosCost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
184 /*CompressionCost=*/true);
185 int NegCost =
186 RISCVMatInt::getIntMatCost(Val: APInt(64, -Imm), Size: 64, STI: *Subtarget,
187 /*CompressionCost=*/true);
188 Rewrite = NegCost != PosCost ? NegCost < PosCost : Imm < 0;
189 }
190 if (!Rewrite)
191 break;
192 SDLoc DL(N);
193 // getConstant uniques onto the existing -C node, so it is shared.
194 Result = CurDAG->getNode(Opcode: ISD::SUB, DL, VT, N1: N->getOperand(Num: 0),
195 N2: CurDAG->getConstant(Val: -Imm, DL, VT));
196 break;
197 }
198 }
199
200 if (Result) {
201 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
202 LLVM_DEBUG(N->dump(CurDAG));
203 LLVM_DEBUG(dbgs() << "\nNew: ");
204 LLVM_DEBUG(Result->dump(CurDAG));
205 LLVM_DEBUG(dbgs() << "\n");
206
207 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
208 MadeChange = true;
209 }
210 }
211
212 if (MadeChange)
213 CurDAG->RemoveDeadNodes();
214}
215
216void RISCVDAGToDAGISel::PostprocessISelDAG() {
217 HandleSDNode Dummy(CurDAG->getRoot());
218 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
219
220 bool MadeChange = false;
221 while (Position != CurDAG->allnodes_begin()) {
222 SDNode *N = &*--Position;
223 // Skip dead nodes and any non-machine opcodes.
224 if (N->use_empty() || !N->isMachineOpcode())
225 continue;
226
227 MadeChange |= doPeepholeSExtW(Node: N);
228
229 // FIXME: This is here only because the VMerge transform doesn't
230 // know how to handle masked true inputs. Once that has been moved
231 // to post-ISEL, this can be deleted as well.
232 MadeChange |= doPeepholeMaskedRVV(Node: cast<MachineSDNode>(Val: N));
233 }
234
235 CurDAG->setRoot(Dummy.getValue());
236
237 // After we're done with everything else, convert IMPLICIT_DEF
238 // passthru operands to NoRegister. This is required to workaround
239 // an optimization deficiency in MachineCSE. This really should
240 // be merged back into each of the patterns (i.e. there's no good
241 // reason not to go directly to NoReg), but is being done this way
242 // to allow easy backporting.
243 MadeChange |= doPeepholeNoRegPassThru();
244
245 if (MadeChange)
246 CurDAG->RemoveDeadNodes();
247}
248
249static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
250 RISCVMatInt::InstSeq &Seq) {
251 SDValue SrcReg = CurDAG->getRegister(Reg: RISCV::X0, VT);
252 for (const RISCVMatInt::Inst &Inst : Seq) {
253 SDValue SDImm = CurDAG->getSignedTargetConstant(Val: Inst.getImm(), DL, VT);
254 SDNode *Result = nullptr;
255 switch (Inst.getOpndKind()) {
256 case RISCVMatInt::Imm:
257 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SDImm);
258 break;
259 case RISCVMatInt::RegX0:
260 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg,
261 Op2: CurDAG->getRegister(Reg: RISCV::X0, VT));
262 break;
263 case RISCVMatInt::RegReg:
264 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SrcReg);
265 break;
266 case RISCVMatInt::RegImm:
267 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SDImm);
268 break;
269 }
270
271 // Only the first instruction has X0 as its source.
272 SrcReg = SDValue(Result, 0);
273 }
274
275 return SrcReg;
276}
277
278static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
279 int64_t Imm, const RISCVSubtarget &Subtarget) {
280 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
281
282 // Use a rematerializable pseudo instruction for short sequences if enabled.
283 if (Seq.size() == 2 && Subtarget.getCLOpts().use_rematerializable_movimm)
284 return SDValue(
285 CurDAG->getMachineNode(Opcode: RISCV::PseudoMovImm, dl: DL, VT,
286 Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT)),
287 0);
288
289 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
290 // worst an LUI+ADDIW. This will require an extra register, but avoids a
291 // constant pool.
292 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
293 // low and high 32 bits are the same and bit 31 and 63 are set.
294 if (Seq.size() > 3) {
295 unsigned ShiftAmt, AddOpc;
296 RISCVMatInt::InstSeq SeqLo =
297 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
298 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
299 SDValue Lo = selectImmSeq(CurDAG, DL, VT, Seq&: SeqLo);
300
301 SDValue SLLI = SDValue(
302 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: Lo,
303 Op2: CurDAG->getTargetConstant(Val: ShiftAmt, DL, VT)),
304 0);
305 return SDValue(CurDAG->getMachineNode(Opcode: AddOpc, dl: DL, VT, Op1: Lo, Op2: SLLI), 0);
306 }
307 }
308
309 // Otherwise, use the original sequence.
310 return selectImmSeq(CurDAG, DL, VT, Seq);
311}
312
313void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
314 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
315 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
316 bool IsLoad, MVT *IndexVT) {
317 SDValue Chain = Node->getOperand(Num: 0);
318
319 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Base pointer.
320
321 if (IsStridedOrIndexed) {
322 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Index.
323 if (IndexVT)
324 *IndexVT = Operands.back()->getSimpleValueType(ResNo: 0);
325 }
326
327 if (IsMasked) {
328 SDValue Mask = Node->getOperand(Num: CurOp++);
329 Operands.push_back(Elt: Mask);
330 }
331 SDValue VL;
332 selectVLOp(N: Node->getOperand(Num: CurOp++), VL);
333 Operands.push_back(Elt: VL);
334
335 MVT XLenVT = Subtarget->getXLenVT();
336 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
337 Operands.push_back(Elt: SEWOp);
338
339 // At the IR layer, all the masked load intrinsics have policy operands,
340 // none of the others do. All have passthru operands. For our pseudos,
341 // all loads have policy operands.
342 if (IsLoad) {
343 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
344 if (IsMasked)
345 Policy = Node->getConstantOperandVal(Num: CurOp++);
346 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
347 Operands.push_back(Elt: PolicyOp);
348 }
349
350 Operands.push_back(Elt: Chain); // Chain.
351}
352
353void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked,
354 bool IsStrided) {
355 SDLoc DL(Node);
356 MVT VT = Node->getSimpleValueType(ResNo: 0);
357 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
358 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
359
360 unsigned CurOp = 2;
361 SmallVector<SDValue, 8> Operands;
362
363 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
364
365 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
366 Operands, /*IsLoad=*/true);
367
368 const RISCV::VLSEGPseudo *P =
369 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
370 LMUL: static_cast<unsigned>(LMUL));
371 MachineSDNode *Load =
372 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
373
374 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
375
376 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
377 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
378 CurDAG->RemoveDeadNode(N: Node);
379}
380
381void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, unsigned NF,
382 bool IsMasked) {
383 SDLoc DL(Node);
384 MVT VT = Node->getSimpleValueType(ResNo: 0);
385 MVT XLenVT = Subtarget->getXLenVT();
386 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
387 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
388
389 unsigned CurOp = 2;
390 SmallVector<SDValue, 7> Operands;
391
392 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
393
394 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
395 /*IsStridedOrIndexed*/ false, Operands,
396 /*IsLoad=*/true);
397
398 const RISCV::VLSEGPseudo *P =
399 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
400 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
401 MachineSDNode *Load = CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped,
402 VT2: XLenVT, VT3: MVT::Other, Ops: Operands);
403
404 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
405
406 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0)); // Result
407 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1)); // VL
408 ReplaceUses(F: SDValue(Node, 2), T: SDValue(Load, 2)); // Chain
409 CurDAG->RemoveDeadNode(N: Node);
410}
411
412void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked,
413 bool IsOrdered) {
414 SDLoc DL(Node);
415 MVT VT = Node->getSimpleValueType(ResNo: 0);
416 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
417 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
418
419 unsigned CurOp = 2;
420 SmallVector<SDValue, 8> Operands;
421
422 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
423
424 MVT IndexVT;
425 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
426 /*IsStridedOrIndexed*/ true, Operands,
427 /*IsLoad=*/true, IndexVT: &IndexVT);
428
429#ifndef NDEBUG
430 // Number of element = RVVBitsPerBlock * LMUL / SEW
431 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
432 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
433 if (DecodedLMUL.second)
434 ContainedTyNumElts /= DecodedLMUL.first;
435 else
436 ContainedTyNumElts *= DecodedLMUL.first;
437 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
438 "Element count mismatch");
439#endif
440
441 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
442 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
443 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
444 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
445 "values when XLEN=32");
446 }
447 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
448 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
449 IndexLMUL: static_cast<unsigned>(IndexLMUL));
450 MachineSDNode *Load =
451 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
452
453 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
454
455 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
456 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
457 CurDAG->RemoveDeadNode(N: Node);
458}
459
460void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked,
461 bool IsStrided) {
462 SDLoc DL(Node);
463 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
464 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
465 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
466
467 unsigned CurOp = 2;
468 SmallVector<SDValue, 8> Operands;
469
470 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
471
472 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
473 Operands);
474
475 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
476 NF, Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
477 MachineSDNode *Store =
478 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
479
480 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
481
482 ReplaceNode(F: Node, T: Store);
483}
484
485void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked,
486 bool IsOrdered) {
487 SDLoc DL(Node);
488 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
489 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
490 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
491
492 unsigned CurOp = 2;
493 SmallVector<SDValue, 8> Operands;
494
495 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
496
497 MVT IndexVT;
498 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
499 /*IsStridedOrIndexed*/ true, Operands,
500 /*IsLoad=*/false, IndexVT: &IndexVT);
501
502#ifndef NDEBUG
503 // Number of element = RVVBitsPerBlock * LMUL / SEW
504 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
505 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
506 if (DecodedLMUL.second)
507 ContainedTyNumElts /= DecodedLMUL.first;
508 else
509 ContainedTyNumElts *= DecodedLMUL.first;
510 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
511 "Element count mismatch");
512#endif
513
514 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
515 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
516 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
517 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
518 "values when XLEN=32");
519 }
520 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
521 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
522 IndexLMUL: static_cast<unsigned>(IndexLMUL));
523 MachineSDNode *Store =
524 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
525
526 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
527
528 ReplaceNode(F: Node, T: Store);
529}
530
531void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) {
532 if (!Subtarget->hasVInstructions())
533 return;
534
535 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
536
537 SDLoc DL(Node);
538 MVT XLenVT = Subtarget->getXLenVT();
539
540 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
541
542 assert((IntNo == Intrinsic::riscv_vsetvli ||
543 IntNo == Intrinsic::riscv_vsetvlimax) &&
544 "Unexpected vsetvli intrinsic");
545
546 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
547 unsigned Offset = (VLMax ? 1 : 2);
548
549 assert(Node->getNumOperands() == Offset + 2 &&
550 "Unexpected number of operands");
551
552 unsigned SEW =
553 RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: Offset) & 0x7);
554 RISCVVType::VLMUL VLMul = static_cast<RISCVVType::VLMUL>(
555 Node->getConstantOperandVal(Num: Offset + 1) & 0x7);
556
557 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
558 /*MaskAgnostic*/ true);
559 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
560
561 SDValue VLOperand;
562 unsigned Opcode = RISCV::PseudoVSETVLI;
563 if (auto *C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
564 if (auto VLEN = Subtarget->getRealVLen())
565 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
566 VLMax = true;
567 }
568 if (VLMax || isAllOnesConstant(V: Node->getOperand(Num: 1))) {
569 VLOperand = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
570 Opcode = RISCV::PseudoVSETVLIX0;
571 } else {
572 VLOperand = Node->getOperand(Num: 1);
573
574 if (auto *C = dyn_cast<ConstantSDNode>(Val&: VLOperand)) {
575 uint64_t AVL = C->getZExtValue();
576 if (isUInt<5>(x: AVL)) {
577 SDValue VLImm = CurDAG->getTargetConstant(Val: AVL, DL, VT: XLenVT);
578 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: RISCV::PseudoVSETIVLI, dl: DL,
579 VT: XLenVT, Op1: VLImm, Op2: VTypeIOp));
580 return;
581 }
582 }
583 }
584
585 ReplaceNode(F: Node,
586 T: CurDAG->getMachineNode(Opcode, dl: DL, VT: XLenVT, Op1: VLOperand, Op2: VTypeIOp));
587}
588
589void RISCVDAGToDAGISel::selectXSfmmVSET(SDNode *Node) {
590 if (!Subtarget->hasVendorXSfmmbase())
591 return;
592
593 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
594
595 SDLoc DL(Node);
596 MVT XLenVT = Subtarget->getXLenVT();
597
598 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
599
600 assert((IntNo == Intrinsic::riscv_sf_vsettnt ||
601 IntNo == Intrinsic::riscv_sf_vsettm ||
602 IntNo == Intrinsic::riscv_sf_vsettk) &&
603 "Unexpected XSfmm vset intrinsic");
604
605 unsigned SEW = RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: 2));
606 unsigned Widen = RISCVVType::decodeTWiden(TWiden: Node->getConstantOperandVal(Num: 3));
607 unsigned PseudoOpCode =
608 IntNo == Intrinsic::riscv_sf_vsettnt ? RISCV::PseudoSF_VSETTNT
609 : IntNo == Intrinsic::riscv_sf_vsettm ? RISCV::PseudoSF_VSETTM
610 : RISCV::PseudoSF_VSETTK;
611
612 if (IntNo == Intrinsic::riscv_sf_vsettnt) {
613 unsigned VTypeI = RISCVVType::encodeXSfmmVType(SEW, Widen, AltFmt: 0);
614 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
615
616 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
617 Op1: Node->getOperand(Num: 1), Op2: VTypeIOp));
618 } else {
619 SDValue Log2SEW = CurDAG->getTargetConstant(Val: Log2_32(Value: SEW), DL, VT: XLenVT);
620 SDValue TWiden = CurDAG->getTargetConstant(Val: Widen, DL, VT: XLenVT);
621 ReplaceNode(F: Node,
622 T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
623 Op1: Node->getOperand(Num: 1), Op2: Log2SEW, Op3: TWiden));
624 }
625}
626
627bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
628 MVT VT = Node->getSimpleValueType(ResNo: 0);
629 unsigned Opcode = Node->getOpcode();
630 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
631 "Unexpected opcode");
632 SDLoc DL(Node);
633
634 // For operations of the form (x << C1) op C2, check if we can use
635 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
636 SDValue N0 = Node->getOperand(Num: 0);
637 SDValue N1 = Node->getOperand(Num: 1);
638
639 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
640 if (!Cst)
641 return false;
642
643 int64_t Val = Cst->getSExtValue();
644
645 // Check if immediate can already use ANDI/ORI/XORI.
646 if (isInt<12>(x: Val))
647 return false;
648
649 SDValue Shift = N0;
650
651 // If Val is simm32 and we have a sext_inreg from i32, then the binop
652 // produces at least 33 sign bits. We can peek through the sext_inreg and use
653 // a SLLIW at the end.
654 bool SignExt = false;
655 if (isInt<32>(x: Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
656 N0.hasOneUse() && cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT() == MVT::i32) {
657 SignExt = true;
658 Shift = N0.getOperand(i: 0);
659 }
660
661 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
662 return false;
663
664 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1));
665 if (!ShlCst)
666 return false;
667
668 uint64_t ShAmt = ShlCst->getZExtValue();
669
670 // Make sure that we don't change the operation by removing bits.
671 // This only matters for OR and XOR, AND is unaffected.
672 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(N: ShAmt);
673 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
674 return false;
675
676 int64_t ShiftedVal = Val >> ShAmt;
677 if (!isInt<12>(x: ShiftedVal))
678 return false;
679
680 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
681 if (SignExt && ShAmt >= 32)
682 return false;
683
684 // Ok, we can reorder to get a smaller immediate.
685 unsigned BinOpc;
686 switch (Opcode) {
687 default: llvm_unreachable("Unexpected opcode");
688 case ISD::AND: BinOpc = RISCV::ANDI; break;
689 case ISD::OR: BinOpc = RISCV::ORI; break;
690 case ISD::XOR: BinOpc = RISCV::XORI; break;
691 }
692
693 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
694
695 SDNode *BinOp = CurDAG->getMachineNode(
696 Opcode: BinOpc, dl: DL, VT, Op1: Shift.getOperand(i: 0),
697 Op2: CurDAG->getSignedTargetConstant(Val: ShiftedVal, DL, VT));
698 SDNode *SLLI =
699 CurDAG->getMachineNode(Opcode: ShOpc, dl: DL, VT, Op1: SDValue(BinOp, 0),
700 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
701 ReplaceNode(F: Node, T: SLLI);
702 return true;
703}
704
705bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
706 unsigned Opc;
707
708 if (Subtarget->hasVendorXTHeadBb())
709 Opc = RISCV::TH_EXT;
710 else if (Subtarget->hasVendorXAndesPerf())
711 Opc = RISCV::NDS_BFOS;
712 else if (Subtarget->hasVendorXqcibm())
713 Opc = RISCV::QC_EXT;
714 else
715 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
716 return false;
717
718 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
719 if (!N1C)
720 return false;
721
722 SDValue N0 = Node->getOperand(Num: 0);
723 if (!N0.hasOneUse())
724 return false;
725
726 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
727 const SDLoc &DL, MVT VT) {
728 if (Opc == RISCV::QC_EXT) {
729 // QC.EXT X, width, shamt
730 // shamt is the same as Lsb
731 // width is the number of bits to extract from the Lsb
732 Msb = Msb - Lsb + 1;
733 }
734 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
735 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
736 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
737 };
738
739 SDLoc DL(Node);
740 MVT VT = Node->getSimpleValueType(ResNo: 0);
741 const unsigned RightShAmt = N1C->getZExtValue();
742
743 // Transform (sra (shl X, C1) C2) with C1 < C2
744 // -> (SignedBitfieldExtract X, msb, lsb)
745 if (N0.getOpcode() == ISD::SHL) {
746 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
747 if (!N01C)
748 return false;
749
750 const unsigned LeftShAmt = N01C->getZExtValue();
751 // Make sure that this is a bitfield extraction (i.e., the shift-right
752 // amount can not be less than the left-shift).
753 if (LeftShAmt > RightShAmt)
754 return false;
755
756 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
757 const unsigned Msb = MsbPlusOne - 1;
758 const unsigned Lsb = RightShAmt - LeftShAmt;
759
760 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
761 ReplaceNode(F: Node, T: Sbe);
762 return true;
763 }
764
765 // Transform (sra (sext_inreg X, _), C) ->
766 // (SignedBitfieldExtract X, msb, lsb)
767 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
768 unsigned ExtSize =
769 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
770
771 // ExtSize of 32 should use sraiw via tablegen pattern.
772 if (ExtSize == 32)
773 return false;
774
775 const unsigned Msb = ExtSize - 1;
776 // If the shift-right amount is greater than Msb, it means that extracts
777 // the X[Msb] bit and sign-extend it.
778 const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
779
780 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
781 ReplaceNode(F: Node, T: Sbe);
782 return true;
783 }
784
785 return false;
786}
787
788bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
789 // Only supported with XAndesPerf at the moment.
790 if (!Subtarget->hasVendorXAndesPerf())
791 return false;
792
793 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
794 if (!N1C)
795 return false;
796
797 SDValue N0 = Node->getOperand(Num: 0);
798 if (!N0.hasOneUse())
799 return false;
800
801 auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
802 const SDLoc &DL, MVT VT) {
803 unsigned Opc = RISCV::NDS_BFOS;
804 // If the Lsb is equal to the Msb, then the Lsb should be 0.
805 if (Lsb == Msb)
806 Lsb = 0;
807 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
808 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
809 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
810 };
811
812 SDLoc DL(Node);
813 MVT VT = Node->getSimpleValueType(ResNo: 0);
814 const unsigned RightShAmt = N1C->getZExtValue();
815
816 // Transform (sra (shl X, C1) C2) with C1 > C2
817 // -> (NDS.BFOS X, lsb, msb)
818 if (N0.getOpcode() == ISD::SHL) {
819 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
820 if (!N01C)
821 return false;
822
823 const unsigned LeftShAmt = N01C->getZExtValue();
824 // Make sure that this is a bitfield insertion (i.e., the shift-right
825 // amount should be less than the left-shift).
826 if (LeftShAmt <= RightShAmt)
827 return false;
828
829 const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
830 const unsigned Msb = MsbPlusOne - 1;
831 const unsigned Lsb = LeftShAmt - RightShAmt;
832
833 SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
834 ReplaceNode(F: Node, T: Sbi);
835 return true;
836 }
837
838 return false;
839}
840
841bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
842 const SDLoc &DL, MVT VT,
843 SDValue X, unsigned Msb,
844 unsigned Lsb) {
845 unsigned Opc;
846
847 if (Subtarget->hasVendorXTHeadBb()) {
848 Opc = RISCV::TH_EXTU;
849 } else if (Subtarget->hasVendorXAndesPerf()) {
850 Opc = RISCV::NDS_BFOZ;
851 } else if (Subtarget->hasVendorXqcibm()) {
852 Opc = RISCV::QC_EXTU;
853 // QC.EXTU X, width, shamt
854 // shamt is the same as Lsb
855 // width is the number of bits to extract from the Lsb
856 Msb = Msb - Lsb + 1;
857 } else {
858 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
859 return false;
860 }
861
862 SDNode *Ube = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
863 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
864 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
865 ReplaceNode(F: Node, T: Ube);
866 return true;
867}
868
869bool RISCVDAGToDAGISel::tryUnsignedBitfieldInsertInZero(SDNode *Node,
870 const SDLoc &DL, MVT VT,
871 SDValue X, unsigned Msb,
872 unsigned Lsb) {
873 // Only supported with XAndesPerf at the moment.
874 if (!Subtarget->hasVendorXAndesPerf())
875 return false;
876
877 unsigned Opc = RISCV::NDS_BFOZ;
878
879 // If the Lsb is equal to the Msb, then the Lsb should be 0.
880 if (Lsb == Msb)
881 Lsb = 0;
882 SDNode *Ubi = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
883 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
884 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
885 ReplaceNode(F: Node, T: Ubi);
886 return true;
887}
888
889bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
890 // Target does not support indexed loads.
891 if (!Subtarget->hasVendorXTHeadMemIdx())
892 return false;
893
894 LoadSDNode *Ld = cast<LoadSDNode>(Val: Node);
895 ISD::MemIndexedMode AM = Ld->getAddressingMode();
896 if (AM == ISD::UNINDEXED)
897 return false;
898
899 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Ld->getOffset());
900 if (!C)
901 return false;
902
903 EVT LoadVT = Ld->getMemoryVT();
904 assert((AM == ISD::PRE_INC || AM == ISD::POST_INC) &&
905 "Unexpected addressing mode");
906 bool IsPre = AM == ISD::PRE_INC;
907 bool IsPost = AM == ISD::POST_INC;
908 int64_t Offset = C->getSExtValue();
909
910 // The constants that can be encoded in the THeadMemIdx instructions
911 // are of the form (sign_extend(imm5) << imm2).
912 unsigned Shift;
913 for (Shift = 0; Shift < 4; Shift++)
914 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
915 break;
916
917 // Constant cannot be encoded.
918 if (Shift == 4)
919 return false;
920
921 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
922 unsigned Opcode;
923 if (LoadVT == MVT::i8 && IsPre)
924 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
925 else if (LoadVT == MVT::i8 && IsPost)
926 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
927 else if (LoadVT == MVT::i16 && IsPre)
928 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
929 else if (LoadVT == MVT::i16 && IsPost)
930 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
931 else if (LoadVT == MVT::i32 && IsPre)
932 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
933 else if (LoadVT == MVT::i32 && IsPost)
934 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
935 else if (LoadVT == MVT::i64 && IsPre)
936 Opcode = RISCV::TH_LDIB;
937 else if (LoadVT == MVT::i64 && IsPost)
938 Opcode = RISCV::TH_LDIA;
939 else
940 return false;
941
942 EVT Ty = Ld->getOffset().getValueType();
943 SDValue Ops[] = {
944 Ld->getBasePtr(),
945 CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(Node), VT: Ty),
946 CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(Node), VT: Ty), Ld->getChain()};
947 SDNode *New = CurDAG->getMachineNode(Opcode, dl: SDLoc(Node), VT1: Ld->getValueType(ResNo: 0),
948 VT2: Ld->getValueType(ResNo: 1), VT3: MVT::Other, Ops);
949
950 MachineMemOperand *MemOp = cast<MemSDNode>(Val: Node)->getMemOperand();
951 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: New), NewMemRefs: {MemOp});
952
953 ReplaceNode(F: Node, T: New);
954
955 return true;
956}
957
958static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT,
959 SDValue Lo, SDValue Hi) {
960 SDValue Ops[] = {
961 CurDAG->getTargetConstant(Val: RISCV::GPRPairRegClassID, DL, VT: MVT::i32), Lo,
962 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_even, DL, VT: MVT::i32), Hi,
963 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_odd, DL, VT: MVT::i32)};
964
965 return SDValue(
966 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT, Ops), 0);
967}
968
969// Helper to extract Lo and Hi values from a GPR pair.
970static std::pair<SDValue, SDValue>
971extractGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, SDValue Pair) {
972 SDValue Lo =
973 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL, VT: MVT::i32, Operand: Pair);
974 SDValue Hi =
975 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_odd, DL, VT: MVT::i32, Operand: Pair);
976 return {Lo, Hi};
977}
978
979// Try to match WMACC pattern: ADDD where one operand pair comes from a
980// widening multiply (both results of UMUL_LOHI, SMUL_LOHI, or WMULSU).
981bool RISCVDAGToDAGISel::tryWideningMulAcc(SDNode *Node, const SDLoc &DL) {
982 assert(Node->getOpcode() == RISCVISD::ADDD && "Expected ADDD");
983
984 SDValue Op0Lo = Node->getOperand(Num: 0);
985 SDValue Op0Hi = Node->getOperand(Num: 1);
986 SDValue Op1Lo = Node->getOperand(Num: 2);
987 SDValue Op1Hi = Node->getOperand(Num: 3);
988
989 auto IsSupportedMulWithOneUse = [](SDValue Lo, SDValue Hi) {
990 unsigned Opc = Lo.getOpcode();
991 if (Opc != ISD::UMUL_LOHI && Opc != ISD::SMUL_LOHI &&
992 Opc != RISCVISD::WMULSU)
993 return false;
994 return Lo.getNode() == Hi.getNode() && Lo.getResNo() == 0 &&
995 Hi.getResNo() == 1 && Lo.hasOneUse() && Hi.hasOneUse();
996 };
997
998 SDNode *MulNode = nullptr;
999 SDValue AddLo, AddHi;
1000
1001 // Check if first operand pair is a supported multiply with single use.
1002 if (IsSupportedMulWithOneUse(Op0Lo, Op0Hi)) {
1003 MulNode = Op0Lo.getNode();
1004 AddLo = Op1Lo;
1005 AddHi = Op1Hi;
1006 }
1007 // ADDD is commutative. Check if second operand pair is a supported multiply
1008 // with single use.
1009 else if (IsSupportedMulWithOneUse(Op1Lo, Op1Hi)) {
1010 MulNode = Op1Lo.getNode();
1011 AddLo = Op0Lo;
1012 AddHi = Op0Hi;
1013 } else {
1014 return false;
1015 }
1016
1017 unsigned Opc;
1018 switch (MulNode->getOpcode()) {
1019 default:
1020 llvm_unreachable("Unexpected multiply opcode");
1021 case ISD::UMUL_LOHI:
1022 Opc = RISCV::WMACCU;
1023 break;
1024 case ISD::SMUL_LOHI:
1025 Opc = RISCV::WMACC;
1026 break;
1027 case RISCVISD::WMULSU:
1028 Opc = RISCV::WMACCSU;
1029 break;
1030 }
1031
1032 SDValue Acc = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: AddLo, Hi: AddHi);
1033
1034 // WMACC instruction format: rd, rs1, rs2 (rd is accumulator).
1035 SDValue M0 = MulNode->getOperand(Num: 0);
1036 SDValue M1 = MulNode->getOperand(Num: 1);
1037 MachineSDNode *New =
1038 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Acc, Op2: M0, Op3: M1);
1039
1040 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
1041 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1042 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1043 CurDAG->RemoveDeadNode(N: Node);
1044 return true;
1045}
1046
1047static Register getTileReg(uint64_t TileNum) {
1048 assert(TileNum <= 15 && "Invalid tile number");
1049 return RISCV::T0 + TileNum;
1050}
1051
1052void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
1053 if (!Subtarget->hasVInstructions())
1054 return;
1055
1056 assert(Node->getOpcode() == ISD::INTRINSIC_VOID && "Unexpected opcode");
1057
1058 SDLoc DL(Node);
1059 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
1060
1061 assert((IntNo == Intrinsic::riscv_sf_vc_x_se ||
1062 IntNo == Intrinsic::riscv_sf_vc_i_se) &&
1063 "Unexpected vsetvli intrinsic");
1064
1065 // imm, imm, imm, simm5/scalar, sew, log2lmul, vl
1066 unsigned Log2SEW = Log2_32(Value: Node->getConstantOperandVal(Num: 6));
1067 SDValue SEWOp =
1068 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: Subtarget->getXLenVT());
1069 SmallVector<SDValue, 8> Operands = {Node->getOperand(Num: 2), Node->getOperand(Num: 3),
1070 Node->getOperand(Num: 4), Node->getOperand(Num: 5),
1071 Node->getOperand(Num: 8), SEWOp,
1072 Node->getOperand(Num: 0)};
1073
1074 unsigned Opcode;
1075 auto *LMulSDNode = cast<ConstantSDNode>(Val: Node->getOperand(Num: 7));
1076 switch (LMulSDNode->getSExtValue()) {
1077 case 5:
1078 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
1079 : RISCV::PseudoSF_VC_I_SE_MF8;
1080 break;
1081 case 6:
1082 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
1083 : RISCV::PseudoSF_VC_I_SE_MF4;
1084 break;
1085 case 7:
1086 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
1087 : RISCV::PseudoSF_VC_I_SE_MF2;
1088 break;
1089 case 0:
1090 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
1091 : RISCV::PseudoSF_VC_I_SE_M1;
1092 break;
1093 case 1:
1094 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
1095 : RISCV::PseudoSF_VC_I_SE_M2;
1096 break;
1097 case 2:
1098 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
1099 : RISCV::PseudoSF_VC_I_SE_M4;
1100 break;
1101 case 3:
1102 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
1103 : RISCV::PseudoSF_VC_I_SE_M8;
1104 break;
1105 }
1106
1107 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
1108 Opcode, dl: DL, VT: Node->getSimpleValueType(ResNo: 0), Ops: Operands));
1109}
1110
1111static unsigned getSegInstNF(unsigned Intrinsic) {
1112#define INST_NF_CASE(NAME, NF) \
1113 case Intrinsic::riscv_##NAME##NF: \
1114 return NF;
1115#define INST_NF_CASE_MASK(NAME, NF) \
1116 case Intrinsic::riscv_##NAME##NF##_mask: \
1117 return NF;
1118#define INST_NF_CASE_FF(NAME, NF) \
1119 case Intrinsic::riscv_##NAME##NF##ff: \
1120 return NF;
1121#define INST_NF_CASE_FF_MASK(NAME, NF) \
1122 case Intrinsic::riscv_##NAME##NF##ff_mask: \
1123 return NF;
1124#define INST_ALL_NF_CASE_BASE(MACRO_NAME, NAME) \
1125 MACRO_NAME(NAME, 2) \
1126 MACRO_NAME(NAME, 3) \
1127 MACRO_NAME(NAME, 4) \
1128 MACRO_NAME(NAME, 5) \
1129 MACRO_NAME(NAME, 6) \
1130 MACRO_NAME(NAME, 7) \
1131 MACRO_NAME(NAME, 8)
1132#define INST_ALL_NF_CASE(NAME) \
1133 INST_ALL_NF_CASE_BASE(INST_NF_CASE, NAME) \
1134 INST_ALL_NF_CASE_BASE(INST_NF_CASE_MASK, NAME)
1135#define INST_ALL_NF_CASE_WITH_FF(NAME) \
1136 INST_ALL_NF_CASE(NAME) \
1137 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF, NAME) \
1138 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF_MASK, NAME)
1139 switch (Intrinsic) {
1140 default:
1141 llvm_unreachable("Unexpected segment load/store intrinsic");
1142 INST_ALL_NF_CASE_WITH_FF(vlseg)
1143 INST_ALL_NF_CASE(vlsseg)
1144 INST_ALL_NF_CASE(vloxseg)
1145 INST_ALL_NF_CASE(vluxseg)
1146 INST_ALL_NF_CASE(vsseg)
1147 INST_ALL_NF_CASE(vssseg)
1148 INST_ALL_NF_CASE(vsoxseg)
1149 INST_ALL_NF_CASE(vsuxseg)
1150 }
1151}
1152
1153static bool isApplicableToPLIOrPLUI(int Val) {
1154 // Check if the immediate is packed i8 or i10
1155 int16_t Bit31To16 = Val >> 16;
1156 int16_t Bit15To0 = Val;
1157 int8_t Bit15To8 = Bit15To0 >> 8;
1158 int8_t Bit7To0 = Val;
1159 if (Bit31To16 != Bit15To0)
1160 return false;
1161
1162 return isInt<10>(x: Bit15To0) || isShiftedInt<10, 6>(x: Bit15To0) ||
1163 Bit15To8 == Bit7To0;
1164}
1165
1166void RISCVDAGToDAGISel::Select(SDNode *Node) {
1167 // If we have a custom node, we have already selected.
1168 if (Node->isMachineOpcode()) {
1169 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
1170 Node->setNodeId(-1);
1171 return;
1172 }
1173
1174 // Instruction Selection not handled by the auto-generated tablegen selection
1175 // should be handled here.
1176 unsigned Opcode = Node->getOpcode();
1177 MVT XLenVT = Subtarget->getXLenVT();
1178 SDLoc DL(Node);
1179 MVT VT = Node->getSimpleValueType(ResNo: 0);
1180
1181 bool HasBitTest = Subtarget->hasBEXTILike();
1182
1183 switch (Opcode) {
1184 case ISD::Constant: {
1185 assert(VT == Subtarget->getXLenVT() && "Unexpected VT");
1186 auto *ConstNode = cast<ConstantSDNode>(Val: Node);
1187 if (ConstNode->isZero()) {
1188 SDValue New =
1189 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: RISCV::X0, VT);
1190 ReplaceNode(F: Node, T: New.getNode());
1191 return;
1192 }
1193 int64_t Imm = ConstNode->getSExtValue();
1194 // If only the lower 8 bits are used, try to convert this to a simm6 by
1195 // sign-extending bit 7. This is neutral without the C extension, and
1196 // allows C.LI to be used if C is present.
1197 if (!isInt<8>(x: Imm) && isUInt<8>(x: Imm) && isInt<6>(x: SignExtend64<8>(x: Imm)) &&
1198 hasAllBUsers(Node))
1199 Imm = SignExtend64<8>(x: Imm);
1200 // If the upper XLen-16 bits are not used, try to convert this to a simm12
1201 // by sign extending bit 15.
1202 else if (!isInt<16>(x: Imm) && isUInt<16>(x: Imm) &&
1203 isInt<12>(x: SignExtend64<16>(x: Imm)) && hasAllHUsers(Node))
1204 Imm = SignExtend64<16>(x: Imm);
1205
1206 // If the upper XLen-16 bits are not used, the lower 2 bytes are the same,
1207 // and we can't use li, convert to an xlen splat so we can use pli.b.
1208 if (Subtarget->hasStdExtP() && !isInt<12>(x: Imm) &&
1209 (Imm & 0xff) == ((Imm >> 8) & 0xff) && hasAllHUsers(Node)) {
1210 // Splat the lower 16 bits to XLen. Sign extend for RV32.
1211 uint64_t Splat = Imm & 0xffff;
1212 Splat = (Splat << 16) | Splat;
1213 if (VT == MVT::i64)
1214 Imm = Splat << 32 | Splat;
1215 else
1216 Imm = SignExtend64<32>(x: Splat);
1217 } else {
1218 // If the upper 32-bits are not used try to convert this into a simm32 by
1219 // sign extending bit 32.
1220 if (!isInt<32>(x: Imm) && isUInt<32>(x: Imm) && hasAllWUsers(Node))
1221 Imm = SignExtend64<32>(x: Imm);
1222
1223 if (VT == MVT::i64 && !isInt<12>(x: Imm) && !isShiftedInt<20, 12>(x: Imm) &&
1224 Subtarget->hasStdExtP() && isApplicableToPLIOrPLUI(Val: Imm) &&
1225 hasAllWUsers(Node)) {
1226 // If it's 4 packed 8-bit integers or 2 packed signed 16-bit integers,
1227 // we can simply copy lower 32 bits to higher 32 bits to make it able to
1228 // rematerialize to PLI_B or PLI_H
1229 Imm = ((uint64_t)Imm << 32) | (Imm & 0xFFFFFFFF);
1230 }
1231 }
1232
1233 ReplaceNode(F: Node, T: selectImm(CurDAG, DL, VT, Imm, Subtarget: *Subtarget).getNode());
1234 return;
1235 }
1236 case ISD::ConstantFP: {
1237 const APFloat &APF = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1238
1239 bool Is64Bit = Subtarget->is64Bit();
1240 bool HasZdinx = Subtarget->hasStdExtZdinx();
1241
1242 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
1243 SDValue Imm;
1244 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
1245 // create an integer immediate.
1246 if (APF.isPosZero() || NegZeroF64) {
1247 if (VT == MVT::f64 && HasZdinx && !Is64Bit)
1248 Imm = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::f64);
1249 else
1250 Imm = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
1251 } else {
1252 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
1253 Subtarget: *Subtarget);
1254 }
1255
1256 unsigned Opc;
1257 switch (VT.SimpleTy) {
1258 default:
1259 llvm_unreachable("Unexpected size");
1260 case MVT::bf16:
1261 assert(Subtarget->hasStdExtZfbfmin());
1262 Opc = RISCV::FMV_H_X;
1263 break;
1264 case MVT::f16:
1265 Opc = Subtarget->hasStdExtZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
1266 break;
1267 case MVT::f32:
1268 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
1269 break;
1270 case MVT::f64:
1271 // For RV32, we can't move from a GPR, we need to convert instead. This
1272 // should only happen for +0.0 and -0.0.
1273 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
1274 if (HasZdinx)
1275 Opc = RISCV::COPY;
1276 else
1277 Opc = Is64Bit ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
1278 break;
1279 }
1280
1281 SDNode *Res;
1282 if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
1283 Res =
1284 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_16, DL, VT, Operand: Imm).getNode();
1285 } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
1286 Res =
1287 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_32, DL, VT, Operand: Imm).getNode();
1288 } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
1289 Res = CurDAG->getMachineNode(
1290 Opcode: Opc, dl: DL, VT, Op1: Imm,
1291 Op2: CurDAG->getTargetConstant(Val: RISCVFPRndMode::RNE, DL, VT: XLenVT));
1292 else
1293 Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: Imm);
1294
1295 // For f64 -0.0, we need to insert a fneg.d idiom.
1296 if (NegZeroF64) {
1297 Opc = RISCV::FSGNJN_D;
1298 if (HasZdinx)
1299 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
1300 Res =
1301 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: SDValue(Res, 0), Op2: SDValue(Res, 0));
1302 }
1303
1304 ReplaceNode(F: Node, T: Res);
1305 return;
1306 }
1307 case RISCVISD::BuildGPRPair:
1308 case RISCVISD::BuildPairF64:
1309 case RISCVISD::BuildPairGPRVec: {
1310 if (Opcode == RISCVISD::BuildPairF64 && !Subtarget->hasStdExtZdinx())
1311 break;
1312
1313 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::BuildPairF64) &&
1314 "BuildPairF64 only handled here on rv32i_zdinx");
1315
1316 SDValue N =
1317 buildGPRPair(CurDAG, DL, VT, Lo: Node->getOperand(Num: 0), Hi: Node->getOperand(Num: 1));
1318 ReplaceNode(F: Node, T: N.getNode());
1319 return;
1320 }
1321 case RISCVISD::SplitGPRPair:
1322 case RISCVISD::SplitF64:
1323 case RISCVISD::SplitGPRVec: {
1324 if (Subtarget->hasStdExtZdinx() || Opcode != RISCVISD::SplitF64) {
1325 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::SplitF64) &&
1326 "SplitF64 only handled here on rv32i_zdinx");
1327
1328 if (!SDValue(Node, 0).use_empty()) {
1329 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL,
1330 VT: Node->getValueType(ResNo: 0),
1331 Operand: Node->getOperand(Num: 0));
1332 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1333 }
1334
1335 if (!SDValue(Node, 1).use_empty()) {
1336 SDValue Hi = CurDAG->getTargetExtractSubreg(
1337 SRIdx: RISCV::sub_gpr_odd, DL, VT: Node->getValueType(ResNo: 1), Operand: Node->getOperand(Num: 0));
1338 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1339 }
1340
1341 CurDAG->RemoveDeadNode(N: Node);
1342 return;
1343 }
1344
1345 if (!Subtarget->hasStdExtZfa())
1346 break;
1347 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
1348 "Unexpected subtarget");
1349
1350 // With Zfa, lower to fmv.x.w and fmvh.x.d.
1351 if (!SDValue(Node, 0).use_empty()) {
1352 SDNode *Lo = CurDAG->getMachineNode(Opcode: RISCV::FMV_X_W_FPR64, dl: DL, VT,
1353 Op1: Node->getOperand(Num: 0));
1354 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Lo, 0));
1355 }
1356 if (!SDValue(Node, 1).use_empty()) {
1357 SDNode *Hi = CurDAG->getMachineNode(Opcode: RISCV::FMVH_X_D, dl: DL, VT,
1358 Op1: Node->getOperand(Num: 0));
1359 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Hi, 0));
1360 }
1361
1362 CurDAG->RemoveDeadNode(N: Node);
1363 return;
1364 }
1365 case ISD::SHL: {
1366 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1367 if (!N1C)
1368 break;
1369 SDValue N0 = Node->getOperand(Num: 0);
1370 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
1371 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1372 break;
1373 unsigned ShAmt = N1C->getZExtValue();
1374 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1375
1376 if (isShiftedMask_64(Value: Mask)) {
1377 unsigned XLen = Subtarget->getXLen();
1378 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1379 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1380 if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
1381 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
1382 // where C2 has 32 leading zeros and C3 trailing zeros.
1383 SDNode *SRLIW = CurDAG->getMachineNode(
1384 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1385 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1386 SDNode *SLLI = CurDAG->getMachineNode(
1387 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1388 Op2: CurDAG->getTargetConstant(Val: TrailingZeros + ShAmt, DL, VT));
1389 ReplaceNode(F: Node, T: SLLI);
1390 return;
1391 }
1392 if (TrailingZeros == 0 && LeadingZeros > ShAmt &&
1393 XLen - LeadingZeros > 11 && LeadingZeros != 32) {
1394 // Optimize (shl (and X, C2), C) -> (srli (slli X, C4), C4-C)
1395 // where C2 has C4 leading zeros and no trailing zeros.
1396 // This is profitable if the "and" was to be lowered to
1397 // (srli (slli X, C4), C4) and not (andi X, C2).
1398 // For "LeadingZeros == 32":
1399 // - with Zba it's just (slli.uw X, C)
1400 // - without Zba a tablegen pattern applies the very same
1401 // transform as we would have done here
1402 SDNode *SLLI = CurDAG->getMachineNode(
1403 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1404 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1405 SDNode *SRLI = CurDAG->getMachineNode(
1406 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1407 Op2: CurDAG->getTargetConstant(Val: LeadingZeros - ShAmt, DL, VT));
1408 ReplaceNode(F: Node, T: SRLI);
1409 return;
1410 }
1411 }
1412 break;
1413 }
1414 case ISD::SRL: {
1415 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1416 if (!N1C)
1417 break;
1418 SDValue N0 = Node->getOperand(Num: 0);
1419 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1420 break;
1421 unsigned ShAmt = N1C->getZExtValue();
1422 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1423
1424 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1425 // 32 leading zeros and C3 trailing zeros.
1426 if (isShiftedMask_64(Value: Mask) && N0.hasOneUse()) {
1427 unsigned XLen = Subtarget->getXLen();
1428 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1429 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1430 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1431 SDNode *SRLIW = CurDAG->getMachineNode(
1432 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1433 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1434 SDNode *SLLI = CurDAG->getMachineNode(
1435 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1436 Op2: CurDAG->getTargetConstant(Val: TrailingZeros - ShAmt, DL, VT));
1437 ReplaceNode(F: Node, T: SLLI);
1438 return;
1439 }
1440 }
1441
1442 // Optimize (srl (and X, C2), C) ->
1443 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1444 // Where C2 is a mask with C3 trailing ones.
1445 // Taking into account that the C2 may have had lower bits unset by
1446 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1447 // This pattern occurs when type legalizing right shifts for types with
1448 // less than XLen bits.
1449 Mask |= maskTrailingOnes<uint64_t>(N: ShAmt);
1450 if (!isMask_64(Value: Mask))
1451 break;
1452 unsigned TrailingOnes = llvm::countr_one(Value: Mask);
1453 if (ShAmt >= TrailingOnes)
1454 break;
1455 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1456 if (TrailingOnes == 32) {
1457 SDNode *SRLI = CurDAG->getMachineNode(
1458 Opcode: Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, dl: DL, VT,
1459 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1460 ReplaceNode(F: Node, T: SRLI);
1461 return;
1462 }
1463
1464 // Only do the remaining transforms if the AND has one use.
1465 if (!N0.hasOneUse())
1466 break;
1467
1468 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1469 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1470 SDNode *BEXTI = CurDAG->getMachineNode(
1471 Opcode: Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, dl: DL, VT,
1472 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1473 ReplaceNode(F: Node, T: BEXTI);
1474 return;
1475 }
1476
1477 const unsigned Msb = TrailingOnes - 1;
1478 const unsigned Lsb = ShAmt;
1479 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0.getOperand(i: 0), Msb, Lsb))
1480 return;
1481
1482 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1483 SDNode *SLLI =
1484 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1485 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1486 SDNode *SRLI = CurDAG->getMachineNode(
1487 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1488 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1489 ReplaceNode(F: Node, T: SRLI);
1490 return;
1491 }
1492 case ISD::SRA: {
1493 if (trySignedBitfieldExtract(Node))
1494 return;
1495
1496 if (trySignedBitfieldInsertInSign(Node))
1497 return;
1498
1499 // Optimize (sra (sext_inreg X, i16), C) ->
1500 // (srai (slli X, (XLen-16), (XLen-16) + C)
1501 // And (sra (sext_inreg X, i8), C) ->
1502 // (srai (slli X, (XLen-8), (XLen-8) + C)
1503 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1504 // This transform matches the code we get without Zbb. The shifts are more
1505 // compressible, and this can help expose CSE opportunities in the sdiv by
1506 // constant optimization.
1507 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1508 if (!N1C)
1509 break;
1510 SDValue N0 = Node->getOperand(Num: 0);
1511 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1512 break;
1513 unsigned ShAmt = N1C->getZExtValue();
1514 unsigned ExtSize =
1515 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
1516 // ExtSize of 32 should use sraiw via tablegen pattern.
1517 if (ExtSize >= 32 || ShAmt >= ExtSize)
1518 break;
1519 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1520 SDNode *SLLI =
1521 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1522 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1523 SDNode *SRAI = CurDAG->getMachineNode(
1524 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1525 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1526 ReplaceNode(F: Node, T: SRAI);
1527 return;
1528 }
1529 case ISD::SIGN_EXTEND_INREG: {
1530 // Optimize (sext_inreg (srl X, C), i8/i16) ->
1531 // (srai (slli X, XLen-ExtSize-C), XLen-ExtSize)
1532 // This is a bitfield extract pattern where we're extracting a signed
1533 // 8-bit or 16-bit field from position C.
1534 SDValue N0 = Node->getOperand(Num: 0);
1535 if (N0.getOpcode() != ISD::SRL || !N0.hasOneUse())
1536 break;
1537
1538 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1539 if (!ShAmtC)
1540 break;
1541
1542 unsigned ExtSize =
1543 cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT().getSizeInBits();
1544 unsigned ShAmt = ShAmtC->getZExtValue();
1545 unsigned XLen = Subtarget->getXLen();
1546
1547 // Only handle types less than 32, and make sure the shift amount is valid.
1548 if (ExtSize >= 32 || ShAmt >= XLen - ExtSize)
1549 break;
1550
1551 unsigned LShAmt = XLen - ExtSize - ShAmt;
1552 SDNode *SLLI =
1553 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1554 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1555 SDNode *SRAI = CurDAG->getMachineNode(
1556 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1557 Op2: CurDAG->getTargetConstant(Val: XLen - ExtSize, DL, VT));
1558 ReplaceNode(F: Node, T: SRAI);
1559 return;
1560 }
1561 case ISD::OR: {
1562 if (tryShrinkShlLogicImm(Node))
1563 return;
1564
1565 break;
1566 }
1567 case ISD::XOR:
1568 if (tryShrinkShlLogicImm(Node))
1569 return;
1570
1571 break;
1572 case ISD::AND: {
1573 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1574 if (!N1C)
1575 break;
1576
1577 SDValue N0 = Node->getOperand(Num: 0);
1578
1579 bool LeftShift = N0.getOpcode() == ISD::SHL;
1580 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1581 auto *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1582 if (!C)
1583 break;
1584 unsigned C2 = C->getZExtValue();
1585 unsigned XLen = Subtarget->getXLen();
1586 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1587
1588 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1589 // shift pair might offer more compression opportunities.
1590 // TODO: We could check for C extension here, but we don't have many lit
1591 // tests with the C extension enabled so not checking gets better
1592 // coverage.
1593 // TODO: What if ANDI faster than shift?
1594 bool IsCANDI = isInt<6>(x: N1C->getSExtValue());
1595
1596 uint64_t C1 = N1C->getZExtValue();
1597
1598 // Clear irrelevant bits in the mask.
1599 if (LeftShift)
1600 C1 &= maskTrailingZeros<uint64_t>(N: C2);
1601 else
1602 C1 &= maskTrailingOnes<uint64_t>(N: XLen - C2);
1603
1604 // Some transforms should only be done if the shift has a single use or
1605 // the AND would become (srli (slli X, 32), 32)
1606 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1607
1608 SDValue X = N0.getOperand(i: 0);
1609
1610 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1611 // with c3 leading zeros.
1612 if (!LeftShift && isMask_64(Value: C1)) {
1613 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1614 if (C2 < Leading) {
1615 // If the number of leading zeros is C2+32 this can be SRLIW.
1616 if (C2 + 32 == Leading) {
1617 SDNode *SRLIW = CurDAG->getMachineNode(
1618 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X, Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1619 ReplaceNode(F: Node, T: SRLIW);
1620 return;
1621 }
1622
1623 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1624 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1625 //
1626 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1627 // legalized and goes through DAG combine.
1628 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1629 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1630 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32) {
1631 SDNode *SRAIW =
1632 CurDAG->getMachineNode(Opcode: RISCV::SRAIW, dl: DL, VT, Op1: X.getOperand(i: 0),
1633 Op2: CurDAG->getTargetConstant(Val: 31, DL, VT));
1634 SDNode *SRLIW = CurDAG->getMachineNode(
1635 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: SDValue(SRAIW, 0),
1636 Op2: CurDAG->getTargetConstant(Val: Leading - 32, DL, VT));
1637 ReplaceNode(F: Node, T: SRLIW);
1638 return;
1639 }
1640
1641 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1642 // available.
1643 // Transform (and (srl x, C2), C1)
1644 // -> (<bfextract> x, msb, lsb)
1645 //
1646 // Make sure to keep this below the SRLIW cases, as we always want to
1647 // prefer the more common instruction.
1648 const unsigned Msb = llvm::bit_width(Value: C1) + C2 - 1;
1649 const unsigned Lsb = C2;
1650 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1651 return;
1652
1653 // (srli (slli x, c3-c2), c3).
1654 // Skip if we could use (zext.w (sraiw X, C2)).
1655 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1656 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1657 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32;
1658 // Also Skip if we can use bexti or th.tst.
1659 Skip |= HasBitTest && Leading == XLen - 1;
1660 if (OneUseOrZExtW && !Skip) {
1661 SDNode *SLLI = CurDAG->getMachineNode(
1662 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1663 Op2: CurDAG->getTargetConstant(Val: Leading - C2, DL, VT));
1664 SDNode *SRLI = CurDAG->getMachineNode(
1665 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1666 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1667 ReplaceNode(F: Node, T: SRLI);
1668 return;
1669 }
1670 }
1671 }
1672
1673 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1674 // shifted by c2 bits with c3 leading zeros.
1675 if (LeftShift && isShiftedMask_64(Value: C1)) {
1676 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1677
1678 if (C2 + Leading < XLen &&
1679 C1 == (maskTrailingOnes<uint64_t>(N: XLen - (C2 + Leading)) << C2)) {
1680 // Use slli.uw when possible.
1681 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1682 SDNode *SLLI_UW =
1683 CurDAG->getMachineNode(Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: X,
1684 Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1685 ReplaceNode(F: Node, T: SLLI_UW);
1686 return;
1687 }
1688
1689 // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
1690 // available.
1691 // Transform (and (shl x, c2), c1)
1692 // -> (<bfinsert> x, msb, lsb)
1693 // e.g.
1694 // (and (shl x, 12), 0x00fff000)
1695 // If XLen = 32 and C2 = 12, then
1696 // Msb = 32 - 8 - 1 = 23 and Lsb = 12
1697 const unsigned Msb = XLen - Leading - 1;
1698 const unsigned Lsb = C2;
1699 if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
1700 return;
1701
1702 if (OneUseOrZExtW && !IsCANDI) {
1703 // (packh x0, X)
1704 if (Subtarget->hasStdExtZbkb() && C1 == 0xff00 && C2 == 8) {
1705 SDNode *PACKH = CurDAG->getMachineNode(
1706 Opcode: RISCV::PACKH, dl: DL, VT,
1707 Op1: CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT()), Op2: X);
1708 ReplaceNode(F: Node, T: PACKH);
1709 return;
1710 }
1711 // (srli (slli c2+c3), c3)
1712 SDNode *SLLI = CurDAG->getMachineNode(
1713 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1714 Op2: CurDAG->getTargetConstant(Val: C2 + Leading, DL, VT));
1715 SDNode *SRLI = CurDAG->getMachineNode(
1716 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1717 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1718 ReplaceNode(F: Node, T: SRLI);
1719 return;
1720 }
1721 }
1722 }
1723
1724 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1725 // shifted mask with c2 leading zeros and c3 trailing zeros.
1726 if (!LeftShift && isShiftedMask_64(Value: C1)) {
1727 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1728 unsigned Trailing = llvm::countr_zero(Val: C1);
1729 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1730 !IsCANDI) {
1731 unsigned SrliOpc = RISCV::SRLI;
1732 // If the input is zexti32 we should use SRLIW.
1733 if (X.getOpcode() == ISD::AND &&
1734 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1735 X.getConstantOperandVal(i: 1) == UINT64_C(0xFFFFFFFF)) {
1736 SrliOpc = RISCV::SRLIW;
1737 X = X.getOperand(i: 0);
1738 }
1739 SDNode *SRLI = CurDAG->getMachineNode(
1740 Opcode: SrliOpc, dl: DL, VT, Op1: X,
1741 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1742 SDNode *SLLI = CurDAG->getMachineNode(
1743 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1744 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1745 ReplaceNode(F: Node, T: SLLI);
1746 return;
1747 }
1748 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1749 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1750 OneUseOrZExtW && !IsCANDI) {
1751 SDNode *SRLIW = CurDAG->getMachineNode(
1752 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1753 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1754 SDNode *SLLI = CurDAG->getMachineNode(
1755 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1756 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1757 ReplaceNode(F: Node, T: SLLI);
1758 return;
1759 }
1760 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1761 if (Trailing > 0 && Leading + Trailing == 32 && C2 + Trailing < XLen &&
1762 OneUseOrZExtW && Subtarget->hasStdExtZba()) {
1763 SDNode *SRLI = CurDAG->getMachineNode(
1764 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1765 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1766 SDNode *SLLI_UW = CurDAG->getMachineNode(
1767 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1768 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1769 ReplaceNode(F: Node, T: SLLI_UW);
1770 return;
1771 }
1772 }
1773
1774 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1775 // shifted mask with no leading zeros and c3 trailing zeros.
1776 if (LeftShift && isShiftedMask_64(Value: C1)) {
1777 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1778 unsigned Trailing = llvm::countr_zero(Val: C1);
1779 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1780 SDNode *SRLI = CurDAG->getMachineNode(
1781 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1782 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1783 SDNode *SLLI = CurDAG->getMachineNode(
1784 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1785 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1786 ReplaceNode(F: Node, T: SLLI);
1787 return;
1788 }
1789 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1790 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1791 SDNode *SRLIW = CurDAG->getMachineNode(
1792 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1793 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1794 SDNode *SLLI = CurDAG->getMachineNode(
1795 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1796 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1797 ReplaceNode(F: Node, T: SLLI);
1798 return;
1799 }
1800
1801 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1802 if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
1803 Subtarget->hasStdExtZba()) {
1804 SDNode *SRLI = CurDAG->getMachineNode(
1805 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1806 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1807 SDNode *SLLI_UW = CurDAG->getMachineNode(
1808 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1809 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1810 ReplaceNode(F: Node, T: SLLI_UW);
1811 return;
1812 }
1813 }
1814 }
1815
1816 const uint64_t C1 = N1C->getZExtValue();
1817
1818 if (N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
1819 N0.hasOneUse()) {
1820 unsigned C2 = N0.getConstantOperandVal(i: 1);
1821 unsigned XLen = Subtarget->getXLen();
1822 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1823
1824 SDValue X = N0.getOperand(i: 0);
1825
1826 // Prefer SRAIW + ANDI when possible.
1827 bool Skip = C2 > 32 && isInt<12>(x: N1C->getSExtValue()) &&
1828 X.getOpcode() == ISD::SHL &&
1829 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1830 X.getConstantOperandVal(i: 1) == 32;
1831 // Turn (and (sra x, c2), c1) -> (srli (srai x, c2-c3), c3) if c1 is a
1832 // mask with c3 leading zeros and c2 is larger than c3.
1833 if (isMask_64(Value: C1) && !Skip) {
1834 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1835 if (C2 > Leading) {
1836 SDNode *SRAI = CurDAG->getMachineNode(
1837 Opcode: RISCV::SRAI, dl: DL, VT, Op1: X,
1838 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1839 SDNode *SRLI = CurDAG->getMachineNode(
1840 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1841 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1842 ReplaceNode(F: Node, T: SRLI);
1843 return;
1844 }
1845 }
1846
1847 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
1848 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
1849 // use (slli (srli (srai y, c2 - c3), c3 + c4), c4).
1850 if (isShiftedMask_64(Value: C1) && !Skip) {
1851 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1852 unsigned Trailing = llvm::countr_zero(Val: C1);
1853 if (C2 > Leading && Leading > 0 && Trailing > 0) {
1854 SDNode *SRAI = CurDAG->getMachineNode(
1855 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1856 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1857 SDNode *SRLI = CurDAG->getMachineNode(
1858 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1859 Op2: CurDAG->getTargetConstant(Val: Leading + Trailing, DL, VT));
1860 SDNode *SLLI = CurDAG->getMachineNode(
1861 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1862 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1863 ReplaceNode(F: Node, T: SLLI);
1864 return;
1865 }
1866 }
1867 }
1868
1869 // If C1 masks off the upper bits only (but can't be formed as an
1870 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1871 // available.
1872 // Transform (and x, C1)
1873 // -> (<bfextract> x, msb, lsb)
1874 if (isMask_64(Value: C1) && !isInt<12>(x: N1C->getSExtValue()) &&
1875 !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
1876 !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
1877 const unsigned Msb = llvm::bit_width(Value: C1) - 1;
1878 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0, Msb, Lsb: 0))
1879 return;
1880 }
1881
1882 if (tryShrinkShlLogicImm(Node))
1883 return;
1884
1885 break;
1886 }
1887 case ISD::MUL: {
1888 // Special case for calculating (mul (and X, C2), C1) where the full product
1889 // fits in XLen bits. We can shift X left by the number of leading zeros in
1890 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1891 // product has XLen trailing zeros, putting it in the output of MULHU. This
1892 // can avoid materializing a constant in a register for C2.
1893
1894 // RHS should be a constant.
1895 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1896 if (!N1C || !N1C->hasOneUse())
1897 break;
1898
1899 // LHS should be an AND with constant.
1900 SDValue N0 = Node->getOperand(Num: 0);
1901 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1902 break;
1903
1904 uint64_t C2 = N0.getConstantOperandVal(i: 1);
1905
1906 // Constant should be a mask.
1907 if (!isMask_64(Value: C2))
1908 break;
1909
1910 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1911 // multiple users or the constant is a simm12. This prevents inserting a
1912 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1913 // make it more costly to materialize. Otherwise, using a SLLI might allow
1914 // it to be compressed.
1915 bool IsANDIOrZExt =
1916 isInt<12>(x: C2) ||
1917 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1918 // With XTHeadBb, we can use TH.EXTU.
1919 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1920 if (IsANDIOrZExt && (isInt<12>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1921 break;
1922 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1923 // the constant is a simm32.
1924 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1925 // With XTHeadBb, we can use TH.EXTU.
1926 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1927 if (IsZExtW && (isInt<32>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1928 break;
1929
1930 // We need to shift left the AND input and C1 by a total of XLen bits.
1931
1932 // How far left do we need to shift the AND input?
1933 unsigned XLen = Subtarget->getXLen();
1934 unsigned LeadingZeros = XLen - llvm::bit_width(Value: C2);
1935
1936 // The constant gets shifted by the remaining amount unless that would
1937 // shift bits out.
1938 uint64_t C1 = N1C->getZExtValue();
1939 unsigned ConstantShift = XLen - LeadingZeros;
1940 if (ConstantShift > (XLen - llvm::bit_width(Value: C1)))
1941 break;
1942
1943 uint64_t ShiftedC1 = C1 << ConstantShift;
1944 // If this RV32, we need to sign extend the constant.
1945 if (XLen == 32)
1946 ShiftedC1 = SignExtend64<32>(x: ShiftedC1);
1947
1948 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1949 SDNode *Imm = selectImm(CurDAG, DL, VT, Imm: ShiftedC1, Subtarget: *Subtarget).getNode();
1950 SDNode *SLLI =
1951 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1952 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1953 SDNode *MULHU = CurDAG->getMachineNode(Opcode: RISCV::MULHU, dl: DL, VT,
1954 Op1: SDValue(SLLI, 0), Op2: SDValue(Imm, 0));
1955 ReplaceNode(F: Node, T: MULHU);
1956 return;
1957 }
1958 case ISD::SMUL_LOHI:
1959 case ISD::UMUL_LOHI:
1960 case RISCVISD::WMULSU:
1961 case RISCVISD::WADD:
1962 case RISCVISD::WSUB:
1963 case RISCVISD::WADDU:
1964 case RISCVISD::WSUBU: {
1965 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1966 "Unexpected opcode");
1967
1968 unsigned Opc;
1969 switch (Node->getOpcode()) {
1970 default:
1971 llvm_unreachable("Unexpected opcode");
1972 case ISD::SMUL_LOHI:
1973 Opc = RISCV::WMUL;
1974 break;
1975 case ISD::UMUL_LOHI:
1976 Opc = RISCV::WMULU;
1977 break;
1978 case RISCVISD::WMULSU:
1979 Opc = RISCV::WMULSU;
1980 break;
1981 case RISCVISD::WADD:
1982 Opc = RISCV::WADD;
1983 break;
1984 case RISCVISD::WSUB:
1985 Opc = RISCV::WSUB;
1986 break;
1987 case RISCVISD::WADDU:
1988 Opc = RISCV::WADDU;
1989 break;
1990 case RISCVISD::WSUBU:
1991 Opc = RISCV::WSUBU;
1992 break;
1993 }
1994
1995 SDNode *Result = CurDAG->getMachineNode(
1996 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Node->getOperand(Num: 0), Op2: Node->getOperand(Num: 1));
1997
1998 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(Result, 0));
1999 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2000 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2001 CurDAG->RemoveDeadNode(N: Node);
2002 return;
2003 }
2004 case RISCVISD::WSLL:
2005 case RISCVISD::WSLA: {
2006 // Custom select WSLL/WSLA for RV32P.
2007 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
2008 "Unexpected opcode");
2009
2010 bool IsSigned = Node->getOpcode() == RISCVISD::WSLA;
2011
2012 SDValue ShAmt = Node->getOperand(Num: 1);
2013
2014 unsigned Opc;
2015
2016 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShAmt);
2017 if (ShAmtC && ShAmtC->getZExtValue() < 64) {
2018 Opc = IsSigned ? RISCV::WSLAI : RISCV::WSLLI;
2019 ShAmt = CurDAG->getTargetConstant(Val: ShAmtC->getZExtValue(), DL, VT: XLenVT);
2020 } else {
2021 Opc = IsSigned ? RISCV::WSLA : RISCV::WSLL;
2022 }
2023
2024 SDNode *WShift = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped,
2025 Op1: Node->getOperand(Num: 0), Op2: ShAmt);
2026
2027 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(WShift, 0));
2028 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2029 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2030 CurDAG->RemoveDeadNode(N: Node);
2031 return;
2032 }
2033 case ISD::LOAD: {
2034 if (tryIndexedLoad(Node))
2035 return;
2036
2037 if (Subtarget->hasVendorXCVmem() && !Subtarget->is64Bit()) {
2038 // We match post-incrementing load here
2039 LoadSDNode *Load = cast<LoadSDNode>(Val: Node);
2040 if (Load->getAddressingMode() != ISD::POST_INC)
2041 break;
2042
2043 SDValue Chain = Node->getOperand(Num: 0);
2044 SDValue Base = Node->getOperand(Num: 1);
2045 SDValue Offset = Node->getOperand(Num: 2);
2046
2047 bool Simm12 = false;
2048 bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
2049
2050 if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Val&: Offset)) {
2051 int ConstantVal = ConstantOffset->getSExtValue();
2052 Simm12 = isInt<12>(x: ConstantVal);
2053 if (Simm12)
2054 Offset = CurDAG->getSignedTargetConstant(Val: ConstantVal, DL: SDLoc(Offset),
2055 VT: Offset.getValueType());
2056 }
2057
2058 unsigned Opcode = 0;
2059 switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
2060 case MVT::i8:
2061 if (Simm12 && SignExtend)
2062 Opcode = RISCV::CV_LB_ri_inc;
2063 else if (Simm12 && !SignExtend)
2064 Opcode = RISCV::CV_LBU_ri_inc;
2065 else if (!Simm12 && SignExtend)
2066 Opcode = RISCV::CV_LB_rr_inc;
2067 else
2068 Opcode = RISCV::CV_LBU_rr_inc;
2069 break;
2070 case MVT::i16:
2071 if (Simm12 && SignExtend)
2072 Opcode = RISCV::CV_LH_ri_inc;
2073 else if (Simm12 && !SignExtend)
2074 Opcode = RISCV::CV_LHU_ri_inc;
2075 else if (!Simm12 && SignExtend)
2076 Opcode = RISCV::CV_LH_rr_inc;
2077 else
2078 Opcode = RISCV::CV_LHU_rr_inc;
2079 break;
2080 case MVT::i32:
2081 if (Simm12)
2082 Opcode = RISCV::CV_LW_ri_inc;
2083 else
2084 Opcode = RISCV::CV_LW_rr_inc;
2085 break;
2086 default:
2087 break;
2088 }
2089 if (!Opcode)
2090 break;
2091
2092 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode, dl: DL, VT1: XLenVT, VT2: XLenVT,
2093 VT3: Chain.getSimpleValueType(), Op1: Base,
2094 Op2: Offset, Op3: Chain));
2095 return;
2096 }
2097 break;
2098 }
2099 case RISCVISD::LD_RV32: {
2100 assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd");
2101
2102 SDValue Base, Offset;
2103 SDValue Chain = Node->getOperand(Num: 0);
2104 SDValue Addr = Node->getOperand(Num: 1);
2105 SelectAddrRegImm(Addr, Base, Offset);
2106
2107 SDValue Ops[] = {Base, Offset, Chain};
2108 MachineSDNode *New = CurDAG->getMachineNode(
2109 Opcode: RISCV::LD_RV32, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2110 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2111 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2112 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2113 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2114 ReplaceUses(F: SDValue(Node, 2), T: SDValue(New, 1));
2115 CurDAG->RemoveDeadNode(N: Node);
2116 return;
2117 }
2118 case RISCVISD::SD_RV32: {
2119 SDValue Base, Offset;
2120 SDValue Chain = Node->getOperand(Num: 0);
2121 SDValue Addr = Node->getOperand(Num: 3);
2122 SelectAddrRegImm(Addr, Base, Offset);
2123
2124 SDValue Lo = Node->getOperand(Num: 1);
2125 SDValue Hi = Node->getOperand(Num: 2);
2126
2127 SDValue RegPair;
2128 // Peephole to use X0_Pair for storing zero.
2129 if (isNullConstant(V: Lo) && isNullConstant(V: Hi)) {
2130 RegPair = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2131 } else {
2132 RegPair = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo, Hi);
2133 }
2134
2135 MachineSDNode *New = CurDAG->getMachineNode(Opcode: RISCV::SD_RV32, dl: DL, VT: MVT::Other,
2136 Ops: {RegPair, Base, Offset, Chain});
2137 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2138 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0));
2139 CurDAG->RemoveDeadNode(N: Node);
2140 return;
2141 }
2142 case RISCVISD::MQWACC:
2143 case RISCVISD::MQRWACC:
2144 case RISCVISD::WMACC:
2145 case RISCVISD::WMACCU:
2146 case RISCVISD::WMACCSU: {
2147 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2148 "Unexpected opcode");
2149
2150 SDValue Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Node->getOperand(Num: 0),
2151 Hi: Node->getOperand(Num: 1));
2152 unsigned Opc;
2153 switch (Opcode) {
2154 default:
2155 llvm_unreachable("Unexpected opcode");
2156 case RISCVISD::MQWACC:
2157 Opc = RISCV::MQWACC;
2158 break;
2159 case RISCVISD::MQRWACC:
2160 Opc = RISCV::MQRWACC;
2161 break;
2162 case RISCVISD::WMACC:
2163 Opc = RISCV::WMACC;
2164 break;
2165 case RISCVISD::WMACCU:
2166 Opc = RISCV::WMACCU;
2167 break;
2168 case RISCVISD::WMACCSU:
2169 Opc = RISCV::WMACCSU;
2170 break;
2171 }
2172 MachineSDNode *New = CurDAG->getMachineNode(
2173 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Node->getOperand(Num: 2), Op3: Node->getOperand(Num: 3));
2174 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2175 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2176 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2177 CurDAG->RemoveDeadNode(N: Node);
2178 return;
2179 }
2180 case RISCVISD::ADDD:
2181 // Try to match WMACC pattern: ADDD where one operand pair comes from a
2182 // widening multiply.
2183 if (tryWideningMulAcc(Node, DL))
2184 return;
2185
2186 // Fall through to regular ADDD selection.
2187 [[fallthrough]];
2188 case RISCVISD::SUBD:
2189 case RISCVISD::WADDAU:
2190 case RISCVISD::WSUBAU:
2191 case RISCVISD::WADDA:
2192 case RISCVISD::WSUBA: {
2193 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2194 "Unexpected opcode");
2195
2196 SDValue Op0Lo = Node->getOperand(Num: 0);
2197 SDValue Op0Hi = Node->getOperand(Num: 1);
2198
2199 SDValue Op0;
2200 if (isNullConstant(V: Op0Lo) && isNullConstant(V: Op0Hi)) {
2201 Op0 = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2202 } else {
2203 Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op0Lo, Hi: Op0Hi);
2204 }
2205
2206 SDValue Op1Lo = Node->getOperand(Num: 2);
2207 SDValue Op1Hi = Node->getOperand(Num: 3);
2208
2209 MachineSDNode *New;
2210 if (Opcode == RISCVISD::WADDAU || Opcode == RISCVISD::WSUBAU ||
2211 Opcode == RISCVISD::WADDA || Opcode == RISCVISD::WSUBA) {
2212 // Widening accumulate: Op0 is the accumulator (GPRPair), Op1Lo and Op1Hi
2213 // are the two 32-bit values.
2214 unsigned Opc;
2215 switch (Opcode) {
2216 default:
2217 llvm_unreachable("Unexpected opcode");
2218 case RISCVISD::WADDAU:
2219 Opc = RISCV::WADDAU;
2220 break;
2221 case RISCVISD::WSUBAU:
2222 Opc = RISCV::WSUBAU;
2223 break;
2224 case RISCVISD::WADDA:
2225 Opc = RISCV::WADDA;
2226 break;
2227 case RISCVISD::WSUBA:
2228 Opc = RISCV::WSUBA;
2229 break;
2230 }
2231 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1Lo, Op3: Op1Hi);
2232 } else {
2233 SDValue Op1 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op1Lo, Hi: Op1Hi);
2234
2235 unsigned Opc;
2236 switch (Opcode) {
2237 default:
2238 llvm_unreachable("Unexpected opcode");
2239 case RISCVISD::ADDD:
2240 Opc = RISCV::ADDD;
2241 break;
2242 case RISCVISD::SUBD:
2243 Opc = RISCV::SUBD;
2244 break;
2245 }
2246 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1);
2247 }
2248
2249 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2250 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2251 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2252 CurDAG->RemoveDeadNode(N: Node);
2253 return;
2254 }
2255 case ISD::INTRINSIC_WO_CHAIN: {
2256 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
2257 switch (IntNo) {
2258 // By default we do not custom select any intrinsic.
2259 default:
2260 break;
2261 case Intrinsic::riscv_vmsgeu:
2262 case Intrinsic::riscv_vmsge: {
2263 SDValue Src1 = Node->getOperand(Num: 1);
2264 SDValue Src2 = Node->getOperand(Num: 2);
2265 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
2266 bool IsCmpConstant = false;
2267 bool IsCmpMinimum = false;
2268 // Only custom select scalar second operand.
2269 if (Src2.getValueType() != XLenVT)
2270 break;
2271 // Small constants are handled with patterns.
2272 int64_t CVal = 0;
2273 MVT Src1VT = Src1.getSimpleValueType();
2274 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2275 IsCmpConstant = true;
2276 CVal = C->getSExtValue();
2277 if (CVal >= -15 && CVal <= 16) {
2278 if (!IsUnsigned || CVal != 0)
2279 break;
2280 IsCmpMinimum = true;
2281 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2282 numBits: Src1VT.getScalarSizeInBits())
2283 .getSExtValue()) {
2284 IsCmpMinimum = true;
2285 }
2286 }
2287 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode, VMSGTOpcode;
2288 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2289 default:
2290 llvm_unreachable("Unexpected LMUL!");
2291#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2292 case RISCVVType::lmulenum: \
2293 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2294 : RISCV::PseudoVMSLT_VX_##suffix; \
2295 VMSGTOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix \
2296 : RISCV::PseudoVMSGT_VX_##suffix; \
2297 break;
2298 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2299 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2300 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2301 CASE_VMSLT_OPCODES(LMUL_1, M1)
2302 CASE_VMSLT_OPCODES(LMUL_2, M2)
2303 CASE_VMSLT_OPCODES(LMUL_4, M4)
2304 CASE_VMSLT_OPCODES(LMUL_8, M8)
2305#undef CASE_VMSLT_OPCODES
2306 }
2307 // Mask operations use the LMUL from the mask type.
2308 switch (RISCVTargetLowering::getLMUL(VT)) {
2309 default:
2310 llvm_unreachable("Unexpected LMUL!");
2311#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix) \
2312 case RISCVVType::lmulenum: \
2313 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
2314 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix; \
2315 break;
2316 CASE_VMNAND_VMSET_OPCODES(LMUL_F8, B64)
2317 CASE_VMNAND_VMSET_OPCODES(LMUL_F4, B32)
2318 CASE_VMNAND_VMSET_OPCODES(LMUL_F2, B16)
2319 CASE_VMNAND_VMSET_OPCODES(LMUL_1, B8)
2320 CASE_VMNAND_VMSET_OPCODES(LMUL_2, B4)
2321 CASE_VMNAND_VMSET_OPCODES(LMUL_4, B2)
2322 CASE_VMNAND_VMSET_OPCODES(LMUL_8, B1)
2323#undef CASE_VMNAND_VMSET_OPCODES
2324 }
2325 SDValue SEW = CurDAG->getTargetConstant(
2326 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2327 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2328 SDValue VL;
2329 selectVLOp(N: Node->getOperand(Num: 3), VL);
2330
2331 // If vmsge(u) with minimum value, expand it to vmset.
2332 if (IsCmpMinimum) {
2333 ReplaceNode(F: Node,
2334 T: CurDAG->getMachineNode(Opcode: VMSetOpcode, dl: DL, VT, Op1: VL, Op2: MaskSEW));
2335 return;
2336 }
2337
2338 if (IsCmpConstant) {
2339 SDValue Imm =
2340 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2341
2342 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMSGTOpcode, dl: DL, VT,
2343 Ops: {Src1, Imm, VL, SEW}));
2344 return;
2345 }
2346
2347 // Expand to
2348 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
2349 SDValue Cmp = SDValue(
2350 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2351 0);
2352 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMNANDOpcode, dl: DL, VT,
2353 Ops: {Cmp, Cmp, VL, MaskSEW}));
2354 return;
2355 }
2356 case Intrinsic::riscv_vmsgeu_mask:
2357 case Intrinsic::riscv_vmsge_mask: {
2358 SDValue Src1 = Node->getOperand(Num: 2);
2359 SDValue Src2 = Node->getOperand(Num: 3);
2360 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
2361 bool IsCmpConstant = false;
2362 bool IsCmpMinimum = false;
2363 // Only custom select scalar second operand.
2364 if (Src2.getValueType() != XLenVT)
2365 break;
2366 // Small constants are handled with patterns.
2367 MVT Src1VT = Src1.getSimpleValueType();
2368 int64_t CVal = 0;
2369 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2370 IsCmpConstant = true;
2371 CVal = C->getSExtValue();
2372 if (CVal >= -15 && CVal <= 16) {
2373 if (!IsUnsigned || CVal != 0)
2374 break;
2375 IsCmpMinimum = true;
2376 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2377 numBits: Src1VT.getScalarSizeInBits())
2378 .getSExtValue()) {
2379 IsCmpMinimum = true;
2380 }
2381 }
2382 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
2383 VMOROpcode, VMSGTMaskOpcode;
2384 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2385 default:
2386 llvm_unreachable("Unexpected LMUL!");
2387#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2388 case RISCVVType::lmulenum: \
2389 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2390 : RISCV::PseudoVMSLT_VX_##suffix; \
2391 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
2392 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
2393 VMSGTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix##_MASK \
2394 : RISCV::PseudoVMSGT_VX_##suffix##_MASK; \
2395 break;
2396 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2397 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2398 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2399 CASE_VMSLT_OPCODES(LMUL_1, M1)
2400 CASE_VMSLT_OPCODES(LMUL_2, M2)
2401 CASE_VMSLT_OPCODES(LMUL_4, M4)
2402 CASE_VMSLT_OPCODES(LMUL_8, M8)
2403#undef CASE_VMSLT_OPCODES
2404 }
2405 // Mask operations use the LMUL from the mask type.
2406 switch (RISCVTargetLowering::getLMUL(VT)) {
2407 default:
2408 llvm_unreachable("Unexpected LMUL!");
2409#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
2410 case RISCVVType::lmulenum: \
2411 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
2412 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
2413 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
2414 break;
2415 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, B64)
2416 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, B32)
2417 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, B16)
2418 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, B8)
2419 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, B4)
2420 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, B2)
2421 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, B1)
2422#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
2423 }
2424 SDValue SEW = CurDAG->getTargetConstant(
2425 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2426 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2427 SDValue VL;
2428 selectVLOp(N: Node->getOperand(Num: 5), VL);
2429 SDValue MaskedOff = Node->getOperand(Num: 1);
2430 SDValue Mask = Node->getOperand(Num: 4);
2431
2432 // If vmsge(u) with minimum value, expand it to vmor mask, maskedoff.
2433 if (IsCmpMinimum) {
2434 // We don't need vmor if the MaskedOff and the Mask are the same
2435 // value.
2436 if (Mask == MaskedOff) {
2437 ReplaceUses(F: Node, T: Mask.getNode());
2438 return;
2439 }
2440 ReplaceNode(F: Node,
2441 T: CurDAG->getMachineNode(Opcode: VMOROpcode, dl: DL, VT,
2442 Ops: {Mask, MaskedOff, VL, MaskSEW}));
2443 return;
2444 }
2445
2446 // If the MaskedOff value and the Mask are the same value use
2447 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
2448 // This avoids needing to copy v0 to vd before starting the next sequence.
2449 if (Mask == MaskedOff) {
2450 SDValue Cmp = SDValue(
2451 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2452 0);
2453 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMANDNOpcode, dl: DL, VT,
2454 Ops: {Mask, Cmp, VL, MaskSEW}));
2455 return;
2456 }
2457
2458 SDValue PolicyOp =
2459 CurDAG->getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
2460
2461 if (IsCmpConstant) {
2462 SDValue Imm =
2463 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2464
2465 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
2466 Opcode: VMSGTMaskOpcode, dl: DL, VT,
2467 Ops: {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp}));
2468 return;
2469 }
2470
2471 // Otherwise use
2472 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
2473 // The result is mask undisturbed.
2474 // We use the same instructions to emulate mask agnostic behavior, because
2475 // the agnostic result can be either undisturbed or all 1.
2476 SDValue Cmp = SDValue(CurDAG->getMachineNode(Opcode: VMSLTMaskOpcode, dl: DL, VT,
2477 Ops: {MaskedOff, Src1, Src2, Mask,
2478 VL, SEW, PolicyOp}),
2479 0);
2480 // vmxor.mm vd, vd, v0 is used to update active value.
2481 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMXOROpcode, dl: DL, VT,
2482 Ops: {Cmp, Mask, VL, MaskSEW}));
2483 return;
2484 }
2485 case Intrinsic::riscv_vsetvli:
2486 case Intrinsic::riscv_vsetvlimax:
2487 return selectVSETVLI(Node);
2488 case Intrinsic::riscv_sf_vsettnt:
2489 case Intrinsic::riscv_sf_vsettm:
2490 case Intrinsic::riscv_sf_vsettk:
2491 return selectXSfmmVSET(Node);
2492 }
2493 break;
2494 }
2495 case ISD::INTRINSIC_W_CHAIN: {
2496 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2497 switch (IntNo) {
2498 // By default we do not custom select any intrinsic.
2499 default:
2500 break;
2501 case Intrinsic::riscv_vlseg2:
2502 case Intrinsic::riscv_vlseg3:
2503 case Intrinsic::riscv_vlseg4:
2504 case Intrinsic::riscv_vlseg5:
2505 case Intrinsic::riscv_vlseg6:
2506 case Intrinsic::riscv_vlseg7:
2507 case Intrinsic::riscv_vlseg8: {
2508 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2509 /*IsStrided*/ false);
2510 return;
2511 }
2512 case Intrinsic::riscv_vlseg2_mask:
2513 case Intrinsic::riscv_vlseg3_mask:
2514 case Intrinsic::riscv_vlseg4_mask:
2515 case Intrinsic::riscv_vlseg5_mask:
2516 case Intrinsic::riscv_vlseg6_mask:
2517 case Intrinsic::riscv_vlseg7_mask:
2518 case Intrinsic::riscv_vlseg8_mask: {
2519 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2520 /*IsStrided*/ false);
2521 return;
2522 }
2523 case Intrinsic::riscv_vlsseg2:
2524 case Intrinsic::riscv_vlsseg3:
2525 case Intrinsic::riscv_vlsseg4:
2526 case Intrinsic::riscv_vlsseg5:
2527 case Intrinsic::riscv_vlsseg6:
2528 case Intrinsic::riscv_vlsseg7:
2529 case Intrinsic::riscv_vlsseg8: {
2530 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2531 /*IsStrided*/ true);
2532 return;
2533 }
2534 case Intrinsic::riscv_vlsseg2_mask:
2535 case Intrinsic::riscv_vlsseg3_mask:
2536 case Intrinsic::riscv_vlsseg4_mask:
2537 case Intrinsic::riscv_vlsseg5_mask:
2538 case Intrinsic::riscv_vlsseg6_mask:
2539 case Intrinsic::riscv_vlsseg7_mask:
2540 case Intrinsic::riscv_vlsseg8_mask: {
2541 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2542 /*IsStrided*/ true);
2543 return;
2544 }
2545 case Intrinsic::riscv_vloxseg2:
2546 case Intrinsic::riscv_vloxseg3:
2547 case Intrinsic::riscv_vloxseg4:
2548 case Intrinsic::riscv_vloxseg5:
2549 case Intrinsic::riscv_vloxseg6:
2550 case Intrinsic::riscv_vloxseg7:
2551 case Intrinsic::riscv_vloxseg8:
2552 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2553 /*IsOrdered*/ true);
2554 return;
2555 case Intrinsic::riscv_vluxseg2:
2556 case Intrinsic::riscv_vluxseg3:
2557 case Intrinsic::riscv_vluxseg4:
2558 case Intrinsic::riscv_vluxseg5:
2559 case Intrinsic::riscv_vluxseg6:
2560 case Intrinsic::riscv_vluxseg7:
2561 case Intrinsic::riscv_vluxseg8:
2562 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2563 /*IsOrdered*/ false);
2564 return;
2565 case Intrinsic::riscv_vloxseg2_mask:
2566 case Intrinsic::riscv_vloxseg3_mask:
2567 case Intrinsic::riscv_vloxseg4_mask:
2568 case Intrinsic::riscv_vloxseg5_mask:
2569 case Intrinsic::riscv_vloxseg6_mask:
2570 case Intrinsic::riscv_vloxseg7_mask:
2571 case Intrinsic::riscv_vloxseg8_mask:
2572 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2573 /*IsOrdered*/ true);
2574 return;
2575 case Intrinsic::riscv_vluxseg2_mask:
2576 case Intrinsic::riscv_vluxseg3_mask:
2577 case Intrinsic::riscv_vluxseg4_mask:
2578 case Intrinsic::riscv_vluxseg5_mask:
2579 case Intrinsic::riscv_vluxseg6_mask:
2580 case Intrinsic::riscv_vluxseg7_mask:
2581 case Intrinsic::riscv_vluxseg8_mask:
2582 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2583 /*IsOrdered*/ false);
2584 return;
2585 case Intrinsic::riscv_vlseg8ff:
2586 case Intrinsic::riscv_vlseg7ff:
2587 case Intrinsic::riscv_vlseg6ff:
2588 case Intrinsic::riscv_vlseg5ff:
2589 case Intrinsic::riscv_vlseg4ff:
2590 case Intrinsic::riscv_vlseg3ff:
2591 case Intrinsic::riscv_vlseg2ff: {
2592 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false);
2593 return;
2594 }
2595 case Intrinsic::riscv_vlseg8ff_mask:
2596 case Intrinsic::riscv_vlseg7ff_mask:
2597 case Intrinsic::riscv_vlseg6ff_mask:
2598 case Intrinsic::riscv_vlseg5ff_mask:
2599 case Intrinsic::riscv_vlseg4ff_mask:
2600 case Intrinsic::riscv_vlseg3ff_mask:
2601 case Intrinsic::riscv_vlseg2ff_mask: {
2602 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true);
2603 return;
2604 }
2605 case Intrinsic::riscv_vloxei:
2606 case Intrinsic::riscv_vloxei_mask:
2607 case Intrinsic::riscv_vluxei:
2608 case Intrinsic::riscv_vluxei_mask: {
2609 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
2610 IntNo == Intrinsic::riscv_vluxei_mask;
2611 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
2612 IntNo == Intrinsic::riscv_vloxei_mask;
2613
2614 MVT VT = Node->getSimpleValueType(ResNo: 0);
2615 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2616
2617 unsigned CurOp = 2;
2618 SmallVector<SDValue, 8> Operands;
2619 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2620
2621 MVT IndexVT;
2622 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2623 /*IsStridedOrIndexed*/ true, Operands,
2624 /*IsLoad=*/true, IndexVT: &IndexVT);
2625
2626 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2627 "Element count mismatch");
2628
2629 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2630 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2631 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2632 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2633 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2634 "index values when XLEN=32");
2635 }
2636 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
2637 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
2638 IndexLMUL: static_cast<unsigned>(IndexLMUL));
2639 MachineSDNode *Load =
2640 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2641
2642 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2643
2644 ReplaceNode(F: Node, T: Load);
2645 return;
2646 }
2647 case Intrinsic::riscv_vlm:
2648 case Intrinsic::riscv_vle:
2649 case Intrinsic::riscv_vle_mask:
2650 case Intrinsic::riscv_vlse:
2651 case Intrinsic::riscv_vlse_mask: {
2652 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
2653 IntNo == Intrinsic::riscv_vlse_mask;
2654 bool IsStrided =
2655 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
2656
2657 MVT VT = Node->getSimpleValueType(ResNo: 0);
2658 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2659
2660 // The riscv_vlm intrinsic are always tail agnostic and no passthru
2661 // operand at the IR level. In pseudos, they have both policy and
2662 // passthru operand. The passthru operand is needed to track the
2663 // "tail undefined" state, and the policy is there just for
2664 // for consistency - it will always be "don't care" for the
2665 // unmasked form.
2666 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
2667 unsigned CurOp = 2;
2668 SmallVector<SDValue, 8> Operands;
2669 if (HasPassthruOperand)
2670 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2671 else {
2672 // We eagerly lower to implicit_def (instead of undef), as we
2673 // otherwise fail to select nodes such as: nxv1i1 = undef
2674 SDNode *Passthru =
2675 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
2676 Operands.push_back(Elt: SDValue(Passthru, 0));
2677 }
2678 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2679 Operands, /*IsLoad=*/true);
2680
2681 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2682 const RISCV::VLEPseudo *P =
2683 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
2684 LMUL: static_cast<unsigned>(LMUL));
2685 MachineSDNode *Load =
2686 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2687
2688 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2689
2690 ReplaceNode(F: Node, T: Load);
2691 return;
2692 }
2693 case Intrinsic::riscv_vleff:
2694 case Intrinsic::riscv_vleff_mask: {
2695 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
2696
2697 MVT VT = Node->getSimpleValueType(ResNo: 0);
2698 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2699
2700 unsigned CurOp = 2;
2701 SmallVector<SDValue, 7> Operands;
2702 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2703 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2704 /*IsStridedOrIndexed*/ false, Operands,
2705 /*IsLoad=*/true);
2706
2707 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2708 const RISCV::VLEPseudo *P =
2709 RISCV::getVLEPseudo(Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
2710 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2711 MachineSDNode *Load = CurDAG->getMachineNode(
2712 Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2713 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2714
2715 ReplaceNode(F: Node, T: Load);
2716 return;
2717 }
2718 case Intrinsic::riscv_nds_vln:
2719 case Intrinsic::riscv_nds_vln_mask:
2720 case Intrinsic::riscv_nds_vlnu:
2721 case Intrinsic::riscv_nds_vlnu_mask: {
2722 bool IsMasked = IntNo == Intrinsic::riscv_nds_vln_mask ||
2723 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2724 bool IsUnsigned = IntNo == Intrinsic::riscv_nds_vlnu ||
2725 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2726
2727 MVT VT = Node->getSimpleValueType(ResNo: 0);
2728 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2729 unsigned CurOp = 2;
2730 SmallVector<SDValue, 8> Operands;
2731
2732 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2733 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2734 /*IsStridedOrIndexed=*/false, Operands,
2735 /*IsLoad=*/true);
2736
2737 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2738 const RISCV::NDSVLNPseudo *P = RISCV::getNDSVLNPseudo(
2739 Masked: IsMasked, Unsigned: IsUnsigned, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2740 MachineSDNode *Load =
2741 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2742
2743 if (auto *MemOp = dyn_cast<MemSDNode>(Val: Node))
2744 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {MemOp->getMemOperand()});
2745
2746 ReplaceNode(F: Node, T: Load);
2747 return;
2748 }
2749 }
2750 break;
2751 }
2752 case ISD::INTRINSIC_VOID: {
2753 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2754 switch (IntNo) {
2755 case Intrinsic::riscv_vsseg2:
2756 case Intrinsic::riscv_vsseg3:
2757 case Intrinsic::riscv_vsseg4:
2758 case Intrinsic::riscv_vsseg5:
2759 case Intrinsic::riscv_vsseg6:
2760 case Intrinsic::riscv_vsseg7:
2761 case Intrinsic::riscv_vsseg8: {
2762 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2763 /*IsStrided*/ false);
2764 return;
2765 }
2766 case Intrinsic::riscv_vsseg2_mask:
2767 case Intrinsic::riscv_vsseg3_mask:
2768 case Intrinsic::riscv_vsseg4_mask:
2769 case Intrinsic::riscv_vsseg5_mask:
2770 case Intrinsic::riscv_vsseg6_mask:
2771 case Intrinsic::riscv_vsseg7_mask:
2772 case Intrinsic::riscv_vsseg8_mask: {
2773 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2774 /*IsStrided*/ false);
2775 return;
2776 }
2777 case Intrinsic::riscv_vssseg2:
2778 case Intrinsic::riscv_vssseg3:
2779 case Intrinsic::riscv_vssseg4:
2780 case Intrinsic::riscv_vssseg5:
2781 case Intrinsic::riscv_vssseg6:
2782 case Intrinsic::riscv_vssseg7:
2783 case Intrinsic::riscv_vssseg8: {
2784 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2785 /*IsStrided*/ true);
2786 return;
2787 }
2788 case Intrinsic::riscv_vssseg2_mask:
2789 case Intrinsic::riscv_vssseg3_mask:
2790 case Intrinsic::riscv_vssseg4_mask:
2791 case Intrinsic::riscv_vssseg5_mask:
2792 case Intrinsic::riscv_vssseg6_mask:
2793 case Intrinsic::riscv_vssseg7_mask:
2794 case Intrinsic::riscv_vssseg8_mask: {
2795 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2796 /*IsStrided*/ true);
2797 return;
2798 }
2799 case Intrinsic::riscv_vsoxseg2:
2800 case Intrinsic::riscv_vsoxseg3:
2801 case Intrinsic::riscv_vsoxseg4:
2802 case Intrinsic::riscv_vsoxseg5:
2803 case Intrinsic::riscv_vsoxseg6:
2804 case Intrinsic::riscv_vsoxseg7:
2805 case Intrinsic::riscv_vsoxseg8:
2806 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2807 /*IsOrdered*/ true);
2808 return;
2809 case Intrinsic::riscv_vsuxseg2:
2810 case Intrinsic::riscv_vsuxseg3:
2811 case Intrinsic::riscv_vsuxseg4:
2812 case Intrinsic::riscv_vsuxseg5:
2813 case Intrinsic::riscv_vsuxseg6:
2814 case Intrinsic::riscv_vsuxseg7:
2815 case Intrinsic::riscv_vsuxseg8:
2816 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2817 /*IsOrdered*/ false);
2818 return;
2819 case Intrinsic::riscv_vsoxseg2_mask:
2820 case Intrinsic::riscv_vsoxseg3_mask:
2821 case Intrinsic::riscv_vsoxseg4_mask:
2822 case Intrinsic::riscv_vsoxseg5_mask:
2823 case Intrinsic::riscv_vsoxseg6_mask:
2824 case Intrinsic::riscv_vsoxseg7_mask:
2825 case Intrinsic::riscv_vsoxseg8_mask:
2826 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2827 /*IsOrdered*/ true);
2828 return;
2829 case Intrinsic::riscv_vsuxseg2_mask:
2830 case Intrinsic::riscv_vsuxseg3_mask:
2831 case Intrinsic::riscv_vsuxseg4_mask:
2832 case Intrinsic::riscv_vsuxseg5_mask:
2833 case Intrinsic::riscv_vsuxseg6_mask:
2834 case Intrinsic::riscv_vsuxseg7_mask:
2835 case Intrinsic::riscv_vsuxseg8_mask:
2836 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2837 /*IsOrdered*/ false);
2838 return;
2839 case Intrinsic::riscv_vsoxei:
2840 case Intrinsic::riscv_vsoxei_mask:
2841 case Intrinsic::riscv_vsuxei:
2842 case Intrinsic::riscv_vsuxei_mask: {
2843 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
2844 IntNo == Intrinsic::riscv_vsuxei_mask;
2845 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
2846 IntNo == Intrinsic::riscv_vsoxei_mask;
2847
2848 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2849 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2850
2851 unsigned CurOp = 2;
2852 SmallVector<SDValue, 8> Operands;
2853 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2854
2855 MVT IndexVT;
2856 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2857 /*IsStridedOrIndexed*/ true, Operands,
2858 /*IsLoad=*/false, IndexVT: &IndexVT);
2859
2860 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2861 "Element count mismatch");
2862
2863 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2864 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2865 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2866 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2867 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2868 "index values when XLEN=32");
2869 }
2870 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
2871 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW,
2872 LMUL: static_cast<unsigned>(LMUL), IndexLMUL: static_cast<unsigned>(IndexLMUL));
2873 MachineSDNode *Store =
2874 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2875
2876 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2877
2878 ReplaceNode(F: Node, T: Store);
2879 return;
2880 }
2881 case Intrinsic::riscv_vsm:
2882 case Intrinsic::riscv_vse:
2883 case Intrinsic::riscv_vse_mask:
2884 case Intrinsic::riscv_vsse:
2885 case Intrinsic::riscv_vsse_mask: {
2886 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
2887 IntNo == Intrinsic::riscv_vsse_mask;
2888 bool IsStrided =
2889 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
2890
2891 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2892 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2893
2894 unsigned CurOp = 2;
2895 SmallVector<SDValue, 8> Operands;
2896 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2897
2898 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2899 Operands);
2900
2901 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2902 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
2903 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2904 MachineSDNode *Store =
2905 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2906 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2907
2908 ReplaceNode(F: Node, T: Store);
2909 return;
2910 }
2911 case Intrinsic::riscv_sf_vc_x_se:
2912 case Intrinsic::riscv_sf_vc_i_se:
2913 selectSF_VC_X_SE(Node);
2914 return;
2915 case Intrinsic::riscv_sf_vlte8:
2916 case Intrinsic::riscv_sf_vlte16:
2917 case Intrinsic::riscv_sf_vlte32:
2918 case Intrinsic::riscv_sf_vlte64: {
2919 unsigned Log2SEW;
2920 unsigned PseudoInst;
2921 switch (IntNo) {
2922 case Intrinsic::riscv_sf_vlte8:
2923 PseudoInst = RISCV::PseudoSF_VLTE8;
2924 Log2SEW = 3;
2925 break;
2926 case Intrinsic::riscv_sf_vlte16:
2927 PseudoInst = RISCV::PseudoSF_VLTE16;
2928 Log2SEW = 4;
2929 break;
2930 case Intrinsic::riscv_sf_vlte32:
2931 PseudoInst = RISCV::PseudoSF_VLTE32;
2932 Log2SEW = 5;
2933 break;
2934 case Intrinsic::riscv_sf_vlte64:
2935 PseudoInst = RISCV::PseudoSF_VLTE64;
2936 Log2SEW = 6;
2937 break;
2938 }
2939
2940 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
2941 SDValue TWidenOp = CurDAG->getTargetConstant(Val: 1, DL, VT: XLenVT);
2942 SDValue Operands[] = {Node->getOperand(Num: 2),
2943 Node->getOperand(Num: 3),
2944 Node->getOperand(Num: 4),
2945 SEWOp,
2946 TWidenOp,
2947 Node->getOperand(Num: 0)};
2948
2949 MachineSDNode *TileLoad =
2950 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2951 CurDAG->setNodeMemRefs(N: TileLoad,
2952 NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2953
2954 ReplaceNode(F: Node, T: TileLoad);
2955 return;
2956 }
2957 case Intrinsic::riscv_sf_mm_s_s:
2958 case Intrinsic::riscv_sf_mm_s_u:
2959 case Intrinsic::riscv_sf_mm_u_s:
2960 case Intrinsic::riscv_sf_mm_u_u:
2961 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2962 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2963 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2964 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
2965 case Intrinsic::riscv_sf_mm_f_f: {
2966 bool HasFRM = false;
2967 unsigned PseudoInst;
2968 switch (IntNo) {
2969 case Intrinsic::riscv_sf_mm_s_s:
2970 PseudoInst = RISCV::PseudoSF_MM_S_S;
2971 break;
2972 case Intrinsic::riscv_sf_mm_s_u:
2973 PseudoInst = RISCV::PseudoSF_MM_S_U;
2974 break;
2975 case Intrinsic::riscv_sf_mm_u_s:
2976 PseudoInst = RISCV::PseudoSF_MM_U_S;
2977 break;
2978 case Intrinsic::riscv_sf_mm_u_u:
2979 PseudoInst = RISCV::PseudoSF_MM_U_U;
2980 break;
2981 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2982 PseudoInst = RISCV::PseudoSF_MM_E5M2_E5M2;
2983 HasFRM = true;
2984 break;
2985 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2986 PseudoInst = RISCV::PseudoSF_MM_E5M2_E4M3;
2987 HasFRM = true;
2988 break;
2989 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2990 PseudoInst = RISCV::PseudoSF_MM_E4M3_E5M2;
2991 HasFRM = true;
2992 break;
2993 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
2994 PseudoInst = RISCV::PseudoSF_MM_E4M3_E4M3;
2995 HasFRM = true;
2996 break;
2997 case Intrinsic::riscv_sf_mm_f_f:
2998 if (Node->getOperand(Num: 3).getValueType().getScalarType() == MVT::bf16)
2999 PseudoInst = RISCV::PseudoSF_MM_F_F_ALT;
3000 else
3001 PseudoInst = RISCV::PseudoSF_MM_F_F;
3002 HasFRM = true;
3003 break;
3004 }
3005 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3006 SDValue Op1 = Node->getOperand(Num: 3);
3007 SDValue Op2 = Node->getOperand(Num: 4);
3008 MVT VT = Op1->getSimpleValueType(ResNo: 0);
3009 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3010 SDValue TmOp = Node->getOperand(Num: 5);
3011 SDValue TnOp = Node->getOperand(Num: 6);
3012 SDValue TkOp = Node->getOperand(Num: 7);
3013 SDValue TWidenOp = Node->getOperand(Num: 8);
3014 SDValue Chain = Node->getOperand(Num: 0);
3015
3016 // sf.mm.f.f with sew=32, twiden=2 is invalid
3017 if (IntNo == Intrinsic::riscv_sf_mm_f_f && Log2SEW == 5 &&
3018 TWidenOp->getAsZExtVal() == 2)
3019 reportFatalUsageError(reason: "sf.mm.f.f doesn't support (sew=32, twiden=2)");
3020
3021 SmallVector<SDValue, 10> Operands(
3022 {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Op1, Op2});
3023 if (HasFRM)
3024 Operands.push_back(
3025 Elt: CurDAG->getTargetConstant(Val: RISCVFPRndMode::DYN, DL, VT: XLenVT));
3026 Operands.append(IL: {TmOp, TnOp, TkOp,
3027 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT), TWidenOp,
3028 Chain});
3029
3030 auto *NewNode =
3031 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
3032
3033 ReplaceNode(F: Node, T: NewNode);
3034 return;
3035 }
3036 case Intrinsic::riscv_sf_vtzero_t: {
3037 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3038 SDValue Tm = Node->getOperand(Num: 3);
3039 SDValue Tn = Node->getOperand(Num: 4);
3040 SDValue Log2SEW = Node->getOperand(Num: 5);
3041 SDValue TWiden = Node->getOperand(Num: 6);
3042 SDValue Chain = Node->getOperand(Num: 0);
3043 auto *NewNode = CurDAG->getMachineNode(
3044 Opcode: RISCV::PseudoSF_VTZERO_T, dl: DL, VTs: Node->getVTList(),
3045 Ops: {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Tm, Tn, Log2SEW,
3046 TWiden, Chain});
3047
3048 ReplaceNode(F: Node, T: NewNode);
3049 return;
3050 }
3051 }
3052 break;
3053 }
3054 case ISD::BITCAST: {
3055 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3056 // Just drop bitcasts between vectors if both are fixed or both are
3057 // scalable.
3058 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
3059 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
3060 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3061 CurDAG->RemoveDeadNode(N: Node);
3062 return;
3063 }
3064 if (Subtarget->hasStdExtP()) {
3065 bool Is32BitCast =
3066 (VT == MVT::i32 && (SrcVT == MVT::v4i8 || SrcVT == MVT::v2i16)) ||
3067 (SrcVT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
3068 bool Is64BitCast =
3069 (VT == MVT::i64 && (SrcVT == MVT::v8i8 || SrcVT == MVT::v4i16 ||
3070 SrcVT == MVT::v2i32)) ||
3071 (SrcVT == MVT::i64 &&
3072 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
3073 if (Is32BitCast || Is64BitCast) {
3074 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3075 CurDAG->RemoveDeadNode(N: Node);
3076 return;
3077 }
3078 }
3079 break;
3080 }
3081 case ISD::SPLAT_VECTOR: {
3082 if (!Subtarget->hasStdExtP())
3083 break;
3084 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0))) {
3085 bool IsDoubleWide = Subtarget->isPExtPackedDoubleType(VT);
3086
3087 if (ConstNode->isZero()) {
3088 MCPhysReg X0Reg = IsDoubleWide ? RISCV::X0_Pair : RISCV::X0;
3089 SDValue New =
3090 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: X0Reg, VT);
3091 ReplaceNode(F: Node, T: New.getNode());
3092 return;
3093 }
3094
3095 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3096 APInt Val = ConstNode->getAPIntValue().trunc(width: EltSize);
3097
3098 // Use LI for all ones since it can be compressed to c.li.
3099 if (Val.isAllOnes() && !IsDoubleWide) {
3100 SDNode *NewNode = CurDAG->getMachineNode(
3101 Opcode: RISCV::ADDI, dl: DL, VT, Op1: CurDAG->getRegister(Reg: RISCV::X0, VT),
3102 Op2: CurDAG->getAllOnesConstant(DL, VT: XLenVT, /*IsTarget=*/true));
3103 ReplaceNode(F: Node, T: NewNode);
3104 return;
3105 }
3106
3107 // Find the smallest splat.
3108 if (Val.getBitWidth() > 16 && Val.isSplat(SplatSizeInBits: 16))
3109 Val = Val.trunc(width: 16);
3110 if (Val.getBitWidth() > 8 && Val.isSplat(SplatSizeInBits: 8))
3111 Val = Val.trunc(width: 8);
3112
3113 EltSize = Val.getBitWidth();
3114 int64_t Imm = Val.getSExtValue();
3115
3116 unsigned Opc = 0;
3117 if (EltSize == 8) {
3118 Opc = IsDoubleWide ? RISCV::PLI_DB : RISCV::PLI_B;
3119 } else if (EltSize == 16 && isInt<10>(x: Imm)) {
3120 Opc = IsDoubleWide ? RISCV::PLI_DH : RISCV::PLI_H;
3121 } else if (!IsDoubleWide && EltSize == 32 && isInt<10>(x: Imm)) {
3122 Opc = RISCV::PLI_W;
3123 } else if (EltSize == 16 && isShiftedInt<10, 6>(x: Imm)) {
3124 Opc = IsDoubleWide ? RISCV::PLUI_DH : RISCV::PLUI_H;
3125 Imm = Imm >> 6;
3126 } else if (!IsDoubleWide && EltSize == 32 && isShiftedInt<10, 22>(x: Imm)) {
3127 Opc = RISCV::PLUI_W;
3128 Imm = Imm >> 22;
3129 }
3130
3131 if (Opc) {
3132 SDNode *NewNode = CurDAG->getMachineNode(
3133 Opcode: Opc, dl: DL, VT, Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT: XLenVT));
3134 ReplaceNode(F: Node, T: NewNode);
3135 return;
3136 }
3137 }
3138
3139 break;
3140 }
3141 case ISD::SCALAR_TO_VECTOR:
3142 if (Subtarget->hasStdExtP()) {
3143 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3144 if ((VT == MVT::v2i32 && SrcVT == MVT::i64) ||
3145 (VT == MVT::v4i8 && SrcVT == MVT::i32)) {
3146 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3147 CurDAG->RemoveDeadNode(N: Node);
3148 return;
3149 }
3150 }
3151 break;
3152 case ISD::INSERT_SUBVECTOR:
3153 case RISCVISD::TUPLE_INSERT: {
3154 SDValue V = Node->getOperand(Num: 0);
3155 SDValue SubV = Node->getOperand(Num: 1);
3156 SDLoc DL(SubV);
3157 auto Idx = Node->getConstantOperandVal(Num: 2);
3158 MVT SubVecVT = SubV.getSimpleValueType();
3159
3160 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3161 MVT SubVecContainerVT = SubVecVT;
3162 // Establish the correct scalable-vector types for any fixed-length type.
3163 if (SubVecVT.isFixedLengthVector()) {
3164 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT: SubVecVT);
3165 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
3166 [[maybe_unused]] bool ExactlyVecRegSized =
3167 Subtarget->expandVScale(X: SubVecVT.getSizeInBits())
3168 .isKnownMultipleOf(RHS: Subtarget->expandVScale(X: VecRegSize));
3169 assert(isPowerOf2_64(Subtarget->expandVScale(SubVecVT.getSizeInBits())
3170 .getKnownMinValue()));
3171 assert(Idx == 0 && (ExactlyVecRegSized || V.isUndef()));
3172 }
3173 MVT ContainerVT = VT;
3174 if (VT.isFixedLengthVector())
3175 ContainerVT = TLI.getContainerForFixedLengthVector(VT);
3176
3177 const auto *TRI = Subtarget->getRegisterInfo();
3178 unsigned SubRegIdx;
3179 std::tie(args&: SubRegIdx, args&: Idx) =
3180 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3181 VecVT: ContainerVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3182
3183 // If the Idx hasn't been completely eliminated then this is a subvector
3184 // insert which doesn't naturally align to a vector register. These must
3185 // be handled using instructions to manipulate the vector registers.
3186 if (Idx != 0)
3187 break;
3188
3189 RISCVVType::VLMUL SubVecLMUL =
3190 RISCVTargetLowering::getLMUL(VT: SubVecContainerVT);
3191 [[maybe_unused]] bool IsSubVecPartReg =
3192 SubVecLMUL == RISCVVType::VLMUL::LMUL_F2 ||
3193 SubVecLMUL == RISCVVType::VLMUL::LMUL_F4 ||
3194 SubVecLMUL == RISCVVType::VLMUL::LMUL_F8;
3195 assert((V.getValueType().isRISCVVectorTuple() || !IsSubVecPartReg ||
3196 V.isUndef()) &&
3197 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
3198 "the subvector is smaller than a full-sized register");
3199
3200 // If we haven't set a SubRegIdx, then we must be going between
3201 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
3202 if (SubRegIdx == RISCV::NoSubRegister) {
3203 unsigned InRegClassID =
3204 RISCVTargetLowering::getRegClassIDForVecVT(VT: ContainerVT);
3205 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3206 InRegClassID &&
3207 "Unexpected subvector extraction");
3208 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3209 SDNode *NewNode = CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
3210 dl: DL, VT, Op1: SubV, Op2: RC);
3211 ReplaceNode(F: Node, T: NewNode);
3212 return;
3213 }
3214
3215 SDValue Insert = CurDAG->getTargetInsertSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V, Subreg: SubV);
3216 ReplaceNode(F: Node, T: Insert.getNode());
3217 return;
3218 }
3219 case RISCVISD::TUPLE_CAST: {
3220 // TUPLE_CAST reinterprets a vector tuple as a different tuple type with
3221 // the same register class (same NF and LMUL), just a different minimum
3222 // number of elements per field. The two types occupy identical
3223 // registers, so just forward the operand like we do for same-register-
3224 // class ISD::BITCAST, without emitting a copy.
3225 assert(RISCVTargetLowering::getRegClassIDForVecVT(VT) ==
3226 RISCVTargetLowering::getRegClassIDForVecVT(
3227 Node->getOperand(0).getSimpleValueType()) &&
3228 "Expected input and output of TUPLE_CAST to use the same "
3229 "register class");
3230 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3231 CurDAG->RemoveDeadNode(N: Node);
3232 return;
3233 }
3234 case ISD::EXTRACT_SUBVECTOR:
3235 case RISCVISD::TUPLE_EXTRACT: {
3236 if (Subtarget->hasStdExtP())
3237 break;
3238
3239 SDValue V = Node->getOperand(Num: 0);
3240 auto Idx = Node->getConstantOperandVal(Num: 1);
3241 MVT InVT = V.getSimpleValueType();
3242
3243 SDLoc DL(V);
3244
3245 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3246 MVT SubVecContainerVT = VT;
3247 // Establish the correct scalable-vector types for any fixed-length type.
3248 if (VT.isFixedLengthVector()) {
3249 assert(Idx == 0);
3250 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
3251 }
3252 if (InVT.isFixedLengthVector())
3253 InVT = TLI.getContainerForFixedLengthVector(VT: InVT);
3254
3255 const auto *TRI = Subtarget->getRegisterInfo();
3256 unsigned SubRegIdx;
3257 std::tie(args&: SubRegIdx, args&: Idx) =
3258 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3259 VecVT: InVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3260
3261 // If the Idx hasn't been completely eliminated then this is a subvector
3262 // extract which doesn't naturally align to a vector register. These must
3263 // be handled using instructions to manipulate the vector registers.
3264 if (Idx != 0)
3265 break;
3266
3267 // If we haven't set a SubRegIdx, then we must be going between
3268 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
3269 if (SubRegIdx == RISCV::NoSubRegister) {
3270 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: InVT);
3271 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3272 InRegClassID &&
3273 "Unexpected subvector extraction");
3274 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3275 SDNode *NewNode =
3276 CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT, Op1: V, Op2: RC);
3277 ReplaceNode(F: Node, T: NewNode);
3278 return;
3279 }
3280
3281 SDValue Extract = CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V);
3282 ReplaceNode(F: Node, T: Extract.getNode());
3283 return;
3284 }
3285 case RISCVISD::VMV_S_X_VL:
3286 case RISCVISD::VFMV_S_F_VL:
3287 case RISCVISD::VMV_V_X_VL:
3288 case RISCVISD::VFMV_V_F_VL: {
3289 // Try to match splat of a scalar load to a strided load with stride of x0.
3290 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
3291 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
3292 if (!Node->getOperand(Num: 0).isUndef())
3293 break;
3294 SDValue Src = Node->getOperand(Num: 1);
3295 auto *Ld = dyn_cast<LoadSDNode>(Val&: Src);
3296 // Can't fold load update node because the second
3297 // output is used so that load update node can't be removed.
3298 if (!Ld || Ld->isIndexed())
3299 break;
3300 EVT MemVT = Ld->getMemoryVT();
3301 // The memory VT should be the same size as the element type.
3302 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
3303 break;
3304 if (!IsProfitableToFold(N: Src, U: Node, Root: Node) ||
3305 !IsLegalToFold(N: Src, U: Node, Root: Node, OptLevel: TM.getOptLevel()))
3306 break;
3307
3308 SDValue VL;
3309 if (IsScalarMove) {
3310 // We could deal with more VL if we update the VSETVLI insert pass to
3311 // avoid introducing more VSETVLI.
3312 if (!isOneConstant(V: Node->getOperand(Num: 2)))
3313 break;
3314 selectVLOp(N: Node->getOperand(Num: 2), VL);
3315 } else
3316 selectVLOp(N: Node->getOperand(Num: 2), VL);
3317
3318 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3319 SDValue SEW = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
3320
3321 // If VL=1, then we don't need to do a strided load and can just do a
3322 // regular load.
3323 bool IsStrided = !isOneConstant(V: VL);
3324
3325 // Only do a strided load if we have optimized zero-stride vector load.
3326 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
3327 break;
3328
3329 SmallVector<SDValue> Operands = {
3330 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT), 0),
3331 Ld->getBasePtr()};
3332 if (IsStrided)
3333 Operands.push_back(Elt: CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT));
3334 uint64_t Policy = RISCVVType::MASK_AGNOSTIC | RISCVVType::TAIL_AGNOSTIC;
3335 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
3336 Operands.append(IL: {VL, SEW, PolicyOp, Ld->getChain()});
3337
3338 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
3339 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
3340 /*IsMasked*/ Masked: false, Strided: IsStrided, /*FF*/ false,
3341 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
3342 MachineSDNode *Load =
3343 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, ResultTys: {VT, MVT::Other}, Ops: Operands);
3344 // Update the chain.
3345 ReplaceUses(F: Src.getValue(R: 1), T: SDValue(Load, 1));
3346 // Record the mem-refs
3347 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {Ld->getMemOperand()});
3348 // Replace the splat with the vlse.
3349 ReplaceNode(F: Node, T: Load);
3350 return;
3351 }
3352 case RISCVISD::LPAD_CALL:
3353 case RISCVISD::LPAD_CALL_INDIRECT: {
3354 bool IsIndirect = Opcode == RISCVISD::LPAD_CALL_INDIRECT;
3355 unsigned PseudoOpc = IsIndirect ? RISCV::PseudoCALLIndirectLpadAlign
3356 : RISCV::PseudoCALLLpadAlign;
3357
3358 uint32_t LpadLabel = 0;
3359 if (std::optional<uint32_t> Label =
3360 Subtarget->getCLOpts().landing_pad_label) {
3361 if (!isUInt<20>(x: *Label))
3362 report_fatal_error(reason: "riscv-landing-pad-label=<val>, <val> needs to fit "
3363 "in unsigned 20-bits");
3364 LpadLabel = *Label;
3365 }
3366
3367 // Preserve the argument-register and register-mask operands, between
3368 // Callee and the optional glue, so the pseudo call still reports its
3369 // call-preserved mask to the register allocator.
3370 SmallVector<SDValue, 8> Ops;
3371 Ops.push_back(Elt: Node->getOperand(Num: 1));
3372 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: LpadLabel, DL, VT: XLenVT));
3373
3374 unsigned NumOps = Node->getNumOperands();
3375 bool HasGlue = Node->getGluedNode() != nullptr;
3376 unsigned RegOperandsEnd = HasGlue ? NumOps - 1 : NumOps;
3377 for (unsigned I = 2; I != RegOperandsEnd; ++I)
3378 Ops.push_back(Elt: Node->getOperand(Num: I));
3379
3380 Ops.push_back(Elt: Node->getOperand(Num: 0));
3381 if (HasGlue)
3382 Ops.push_back(Elt: Node->getOperand(Num: NumOps - 1));
3383
3384 ReplaceNode(F: Node,
3385 T: CurDAG->getMachineNode(Opcode: PseudoOpc, dl: DL, VTs: Node->getVTList(), Ops));
3386 return;
3387 }
3388 case ISD::PREFETCH:
3389 // MIPS's prefetch instruction already encodes the hint within the
3390 // instruction itself, so no extra NTL hint is needed.
3391 if (Subtarget->hasVendorXMIPSCBOP())
3392 break;
3393
3394 unsigned Locality = Node->getConstantOperandVal(Num: 3);
3395 if (Locality > 2)
3396 break;
3397
3398 auto *LoadStoreMem = cast<MemSDNode>(Val: Node);
3399 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
3400 MMO->setFlags(MachineMemOperand::MONonTemporal);
3401
3402 int NontemporalLevel = 0;
3403 switch (Locality) {
3404 case 0:
3405 NontemporalLevel = 3; // NTL.ALL
3406 break;
3407 case 1:
3408 NontemporalLevel = 1; // NTL.PALL
3409 break;
3410 case 2:
3411 NontemporalLevel = 0; // NTL.P1
3412 break;
3413 default:
3414 llvm_unreachable("unexpected locality value.");
3415 }
3416
3417 if (NontemporalLevel & 0b1)
3418 MMO->setFlags(MONontemporalBit0);
3419 if (NontemporalLevel & 0b10)
3420 MMO->setFlags(MONontemporalBit1);
3421 break;
3422 }
3423
3424 // Select the default instruction.
3425 SelectCode(N: Node);
3426}
3427
3428bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
3429 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
3430 std::vector<SDValue> &OutOps) {
3431 // Always produce a register and immediate operand, as expected by
3432 // RISCVAsmPrinter::PrintAsmMemoryOperand.
3433 switch (ConstraintID) {
3434 case InlineAsm::ConstraintCode::o:
3435 case InlineAsm::ConstraintCode::m: {
3436 SDValue Op0, Op1;
3437 [[maybe_unused]] bool Found = SelectAddrRegImm(Addr: Op, Base&: Op0, Offset&: Op1);
3438 assert(Found && "SelectAddrRegImm should always succeed");
3439 OutOps.push_back(x: Op0);
3440 OutOps.push_back(x: Op1);
3441 return false;
3442 }
3443 case InlineAsm::ConstraintCode::A:
3444 OutOps.push_back(x: Op);
3445 OutOps.push_back(
3446 x: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget->getXLenVT()));
3447 return false;
3448 default:
3449 report_fatal_error(reason: "Unexpected asm memory constraint " +
3450 InlineAsm::getMemConstraintName(C: ConstraintID));
3451 }
3452
3453 return true;
3454}
3455
3456bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
3457 SDValue &Offset) {
3458 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Addr)) {
3459 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT: Subtarget->getXLenVT());
3460 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: Subtarget->getXLenVT());
3461 return true;
3462 }
3463
3464 return false;
3465}
3466
3467// Fold constant addresses.
3468static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
3469 const MVT VT, const RISCVSubtarget *Subtarget,
3470 SDValue Addr, SDValue &Base, SDValue &Offset,
3471 bool IsPrefetch = false) {
3472 if (!isa<ConstantSDNode>(Val: Addr))
3473 return false;
3474
3475 int64_t CVal = cast<ConstantSDNode>(Val&: Addr)->getSExtValue();
3476
3477 // If the constant is a simm12, we can fold the whole constant and use X0 as
3478 // the base. If the constant can be materialized with LUI+simm12, use LUI as
3479 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
3480 int64_t Lo12 = SignExtend64<12>(x: CVal);
3481 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
3482 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
3483 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3484 return false;
3485 if (Hi) {
3486 int64_t Hi20 = (Hi >> 12) & 0xfffff;
3487 Base = SDValue(
3488 CurDAG->getMachineNode(Opcode: RISCV::LUI, dl: DL, VT,
3489 Op1: CurDAG->getTargetConstant(Val: Hi20, DL, VT)),
3490 0);
3491 } else {
3492 Base = CurDAG->getRegister(Reg: RISCV::X0, VT);
3493 }
3494 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3495 return true;
3496 }
3497
3498 // Ask how constant materialization would handle this constant.
3499 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
3500
3501 // If the last instruction would be an ADDI, we can fold its immediate and
3502 // emit the rest of the sequence as the base.
3503 if (Seq.back().getOpcode() != RISCV::ADDI)
3504 return false;
3505 Lo12 = Seq.back().getImm();
3506 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3507 return false;
3508
3509 // Drop the last instruction.
3510 Seq.pop_back();
3511 assert(!Seq.empty() && "Expected more instructions in sequence");
3512
3513 Base = selectImmSeq(CurDAG, DL, VT, Seq);
3514 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3515 return true;
3516}
3517
3518// Is this ADD instruction only used as the base pointer of scalar loads and
3519// stores?
3520static bool isWorthFoldingAdd(SDValue Add) {
3521 for (auto *User : Add->users()) {
3522 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
3523 User->getOpcode() != RISCVISD::LD_RV32 &&
3524 User->getOpcode() != RISCVISD::SD_RV32 &&
3525 User->getOpcode() != ISD::ATOMIC_LOAD &&
3526 User->getOpcode() != ISD::ATOMIC_STORE)
3527 return false;
3528 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3529 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
3530 VT != MVT::f64)
3531 return false;
3532 // Don't allow stores of the value. It must be used as the address.
3533 if (User->getOpcode() == ISD::STORE &&
3534 cast<StoreSDNode>(Val: User)->getValue() == Add)
3535 return false;
3536 if (User->getOpcode() == ISD::ATOMIC_STORE &&
3537 cast<AtomicSDNode>(Val: User)->getVal() == Add)
3538 return false;
3539 if (User->getOpcode() == RISCVISD::SD_RV32 &&
3540 (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add))
3541 return false;
3542 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
3543 return false;
3544 }
3545
3546 return true;
3547}
3548
3549bool isRegImmLoadOrStore(SDNode *User, SDValue Add) {
3550 switch (User->getOpcode()) {
3551 default:
3552 return false;
3553 case ISD::LOAD:
3554 case RISCVISD::LD_RV32:
3555 case ISD::ATOMIC_LOAD:
3556 break;
3557 case ISD::STORE:
3558 // Don't allow stores of Add. It must only be used as the address.
3559 if (cast<StoreSDNode>(Val: User)->getValue() == Add)
3560 return false;
3561 break;
3562 case RISCVISD::SD_RV32:
3563 // Don't allow stores of Add. It must only be used as the address.
3564 if (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add)
3565 return false;
3566 break;
3567 case ISD::ATOMIC_STORE:
3568 // Don't allow stores of Add. It must only be used as the address.
3569 if (cast<AtomicSDNode>(Val: User)->getVal() == Add)
3570 return false;
3571 break;
3572 }
3573
3574 return true;
3575}
3576
3577// To prevent SelectAddrRegImm from folding offsets that conflict with the
3578// fusion of PseudoMovAddr, check if the offset of every use of a given address
3579// is within the alignment.
3580bool RISCVDAGToDAGISel::areOffsetsWithinAlignment(SDValue Addr,
3581 Align Alignment) {
3582 assert(Addr->getOpcode() == RISCVISD::ADD_LO);
3583 for (auto *User : Addr->users()) {
3584 // If the user is a load or store, then the offset is 0 which is always
3585 // within alignment.
3586 if (isRegImmLoadOrStore(User, Add: Addr))
3587 continue;
3588
3589 if (CurDAG->isBaseWithConstantOffset(Op: SDValue(User, 0))) {
3590 int64_t CVal = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
3591 if (!isInt<12>(x: CVal) || Alignment <= CVal)
3592 return false;
3593
3594 // Make sure all uses are foldable load/stores.
3595 for (auto *AddUser : User->users())
3596 if (!isRegImmLoadOrStore(User: AddUser, Add: SDValue(User, 0)))
3597 return false;
3598
3599 continue;
3600 }
3601
3602 return false;
3603 }
3604
3605 return true;
3606}
3607
3608bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
3609 SDValue &Offset) {
3610 if (SelectAddrFrameIndex(Addr, Base, Offset))
3611 return true;
3612
3613 SDLoc DL(Addr);
3614 MVT VT = Addr.getSimpleValueType();
3615
3616 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
3617 bool CanFold = true;
3618 // Unconditionally fold if operand 1 is not a global address (e.g.
3619 // externsymbol)
3620 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val: Addr.getOperand(i: 1))) {
3621 const DataLayout &DL = CurDAG->getDataLayout();
3622 Align Alignment = commonAlignment(
3623 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3624 if (!areOffsetsWithinAlignment(Addr, Alignment))
3625 CanFold = false;
3626 }
3627 if (CanFold) {
3628 Base = Addr.getOperand(i: 0);
3629 Offset = Addr.getOperand(i: 1);
3630 return true;
3631 }
3632 }
3633
3634 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3635 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3636 if (isInt<12>(x: CVal)) {
3637 Base = Addr.getOperand(i: 0);
3638 if (Base.getOpcode() == RISCVISD::ADD_LO) {
3639 SDValue LoOperand = Base.getOperand(i: 1);
3640 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: LoOperand)) {
3641 // If the Lo in (ADD_LO hi, lo) is a global variable's address
3642 // (its low part, really), then we can rely on the alignment of that
3643 // variable to provide a margin of safety before low part can overflow
3644 // the 12 bits of the load/store offset. Check if CVal falls within
3645 // that margin; if so (low part + CVal) can't overflow.
3646 const DataLayout &DL = CurDAG->getDataLayout();
3647 Align Alignment = commonAlignment(
3648 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3649 if ((CVal == 0 || Alignment > CVal) &&
3650 areOffsetsWithinAlignment(Addr: Base, Alignment)) {
3651 int64_t CombinedOffset = CVal + GA->getOffset();
3652 Base = Base.getOperand(i: 0);
3653 Offset = CurDAG->getTargetGlobalAddress(
3654 GV: GA->getGlobal(), DL: SDLoc(LoOperand), VT: LoOperand.getValueType(),
3655 offset: CombinedOffset, TargetFlags: GA->getTargetFlags());
3656 return true;
3657 }
3658 }
3659 }
3660
3661 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3662 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3663 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3664 return true;
3665 }
3666 }
3667
3668 // Handle ADD with large immediates.
3669 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3670 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3671 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3672
3673 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
3674 // an ADDI for part of the offset and fold the rest into the load/store.
3675 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
3676 if (CVal >= -4096 && CVal <= 4094) {
3677 int64_t Adj = CVal < 0 ? -2048 : 2047;
3678 Base = SDValue(
3679 CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3680 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3681 0);
3682 Offset = CurDAG->getSignedTargetConstant(Val: CVal - Adj, DL, VT);
3683 return true;
3684 }
3685
3686 // For larger immediates, we might be able to save one instruction from
3687 // constant materialization by folding the Lo12 bits of the immediate into
3688 // the address. We should only do this if the ADD is only used by loads and
3689 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
3690 // separately with the full materialized immediate creating extra
3691 // instructions.
3692 if (isWorthFoldingAdd(Add: Addr) &&
3693 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3694 Offset, /*IsPrefetch=*/false)) {
3695 // Insert an ADD instruction with the materialized Hi52 bits.
3696 Base = SDValue(
3697 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3698 0);
3699 return true;
3700 }
3701 }
3702
3703 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3704 /*IsPrefetch=*/false))
3705 return true;
3706
3707 Base = Addr;
3708 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3709 return true;
3710}
3711
3712/// Similar to SelectAddrRegImm, but only matches a register, or a register
3713/// plus a simm12 offset. Doesn't match a FrameIndex or global address, since
3714/// those aren't valid for the callers of this function (e.g. the target of
3715/// an indirect branch).
3716bool RISCVDAGToDAGISel::SelectBrindRegImm(SDValue Addr, SDValue &Base,
3717 SDValue &Offset) {
3718 SDLoc DL(Addr);
3719 MVT VT = Addr.getSimpleValueType();
3720
3721 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3722 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3723 if (isInt<12>(x: CVal)) {
3724 Base = Addr.getOperand(i: 0);
3725 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3726 return true;
3727 }
3728 }
3729
3730 Base = Addr;
3731 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3732 return true;
3733}
3734
3735/// Similar to SelectAddrRegImm, except that the offset is a 26-bit signed
3736/// immediate. This is used by the Qualcomm Xqcilo large offset load/store
3737/// instructions (qc.e.lw/qc.e.sw), whose offset field is 26 bits wide.
3738/// Only matches offsets that do not fit a 12-bit signed immediate, so that
3739/// offsets in the simm12 range keep using the shorter (and possibly
3740/// compressible) standard load/store instructions.
3741bool RISCVDAGToDAGISel::SelectAddrRegImm26(SDValue Addr, SDValue &Base,
3742 SDValue &Offset) {
3743 SDLoc DL(Addr);
3744 MVT VT = Addr.getSimpleValueType();
3745
3746 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3747 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3748 // Fold a 26-bit (but not 12-bit) signed offset directly into the
3749 // load/store.
3750 if (isInt<26>(x: CVal) && !isInt<12>(x: CVal)) {
3751 Base = Addr.getOperand(i: 0);
3752 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3753 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3754 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3755 return true;
3756 }
3757 }
3758
3759 // The offset is just outside the 26-bit range. Split off a small (simm12)
3760 // adjustment with a plain ADDI and fold the remaining 26-bit offset into the
3761 // load/store. A plain ADDI is used (rather than the wide
3762 // qc.e.addi/qc.e.addai) because the adjustment fits simm12: this keeps it a
3763 // short, compressible (c.addi) instruction and is available without Xqcilia.
3764 //
3765 // Skip the split if the address is used other than as a foldable load/store
3766 // base. `isWorthFoldingAdd()` returns true when every user of the add node is
3767 // a scalar load/store using it as an address operand. If it return false, it
3768 // means that some use consumes the add result as a value (e.g. it feeds
3769 // another add, is a stored value, is used in arithmetic) and that use forces
3770 // the add to be materialized into a register.
3771 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)) &&
3772 isWorthFoldingAdd(Add: Addr)) {
3773 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3774 if (!isInt<26>(x: CVal)) {
3775 // check if lw in lui + add + lw combination can be compressed.
3776 // The check here purely based on the immediate value and hopes that
3777 // register allocator would assign a register from a GPRC set so that the
3778 // instruction can get compressed.
3779 bool IsLwCompressable = isShiftedUInt<5, 2>(x: CVal & ((1 << 12) - 1));
3780
3781 int64_t Imm26 = CVal < 0 ? minIntN(N: 26) : maxIntN(N: 26);
3782 int64_t Adj = CVal - Imm26;
3783 // If Adj fits within 6-bits, then both combinations will take 8 bytes
3784 // however c.addi + qc.e.lw/sw will take 1 less cycle. Also, if lw is not
3785 // compressable then both combination would take 10 bytes but again
3786 // addi + qc.e.lw/sw will take 1 less cycle.
3787 if (isInt<6>(x: Adj) || (isInt<12>(x: Adj) && !IsLwCompressable)) {
3788 Base = SDValue(CurDAG->getMachineNode(
3789 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3790 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3791 0);
3792 Offset = CurDAG->getSignedTargetConstant(Val: Imm26, DL, VT);
3793 return true;
3794 }
3795 }
3796 }
3797
3798 // Don't match: let the standard addressing modes handle it.
3799 return false;
3800}
3801
3802/// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
3803bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
3804 SDValue &Offset) {
3805 if (SelectAddrFrameIndex(Addr, Base, Offset))
3806 return true;
3807
3808 SDLoc DL(Addr);
3809 MVT VT = Addr.getSimpleValueType();
3810
3811 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3812 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3813 if (isUInt<9>(x: CVal)) {
3814 Base = Addr.getOperand(i: 0);
3815
3816 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3817 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3818 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3819 return true;
3820 }
3821 }
3822
3823 Base = Addr;
3824 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3825 return true;
3826}
3827
3828/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
3829/// Offset should be all zeros.
3830bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
3831 SDValue &Offset) {
3832 if (SelectAddrFrameIndex(Addr, Base, Offset))
3833 return true;
3834
3835 SDLoc DL(Addr);
3836 MVT VT = Addr.getSimpleValueType();
3837
3838 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3839 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3840 if (isInt<12>(x: CVal)) {
3841 Base = Addr.getOperand(i: 0);
3842
3843 // Early-out if not a valid offset.
3844 if ((CVal & 0b11111) != 0) {
3845 Base = Addr;
3846 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3847 return true;
3848 }
3849
3850 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3851 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3852 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3853 return true;
3854 }
3855 }
3856
3857 // Handle ADD with large immediates.
3858 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3859 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3860 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3861
3862 // Handle immediates in the range [-4096,-2049] or [2017, 4063]. We can save
3863 // one instruction by folding adjustment (-2048 or 2016) into the address.
3864 // The upper bound keeps CVal - 2016 within simm12 ([−2048, 2047]).
3865 if ((-2049 >= CVal && CVal >= -4096) || (4063 >= CVal && CVal >= 2017)) {
3866 int64_t Adj = CVal < 0 ? -2048 : 2016;
3867 int64_t AdjustedOffset = CVal - Adj;
3868 Base =
3869 SDValue(CurDAG->getMachineNode(
3870 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3871 Op2: CurDAG->getSignedTargetConstant(Val: AdjustedOffset, DL, VT)),
3872 0);
3873 Offset = CurDAG->getSignedTargetConstant(Val: Adj, DL, VT);
3874 return true;
3875 }
3876
3877 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3878 Offset, /*IsPrefetch=*/true)) {
3879 // Insert an ADD instruction with the materialized Hi52 bits.
3880 Base = SDValue(
3881 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3882 0);
3883 return true;
3884 }
3885 }
3886
3887 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3888 /*IsPrefetch=*/true))
3889 return true;
3890
3891 Base = Addr;
3892 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3893 return true;
3894}
3895
3896/// Return true if this a load/store that we have a RegRegScale instruction for.
3897static bool isRegRegScaleLoadOrStore(SDNode *User, SDValue Add,
3898 const RISCVSubtarget &Subtarget) {
3899 unsigned UserOpc = User->getOpcode();
3900 if (UserOpc != ISD::LOAD && UserOpc != ISD::STORE)
3901 return false;
3902 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3903 // Zilx only provides indexed loads, so it must not enable reg+reg-scale
3904 // address folding for stores. XTheadMemIdx and Xqcisls have scaled stores.
3905 bool HasScalarIntegerMemIdx =
3906 Subtarget.hasVendorXTHeadMemIdx() || Subtarget.hasVendorXqcisls() ||
3907 (Subtarget.hasStdExtZilx() && UserOpc == ISD::LOAD);
3908 if (!(VT.isScalarInteger() && HasScalarIntegerMemIdx) &&
3909 !((VT == MVT::f32 || VT == MVT::f64) &&
3910 Subtarget.hasVendorXTHeadFMemIdx()))
3911 return false;
3912 // Don't allow stores of the value. It must be used as the address.
3913 if (UserOpc == ISD::STORE && cast<StoreSDNode>(Val: User)->getValue() == Add)
3914 return false;
3915
3916 return true;
3917}
3918
3919/// Is it profitable to fold this Add into RegRegScale load/store. If \p
3920/// Shift is non-null, then we have matched a shl+add. We allow reassociating
3921/// (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2)) if there is a
3922/// single addi and we don't have a SHXADD instruction we could use.
3923/// FIXME: May still need to check how many and what kind of users the SHL has.
3924static bool isWorthFoldingIntoRegRegScale(const RISCVSubtarget &Subtarget,
3925 SDValue Add,
3926 SDValue Shift = SDValue()) {
3927 bool FoundADDI = false;
3928 for (auto *User : Add->users()) {
3929 if (isRegRegScaleLoadOrStore(User, Add, Subtarget))
3930 continue;
3931
3932 // Allow a single ADDI that is used by loads/stores if we matched a shift.
3933 if (!Shift || FoundADDI || User->getOpcode() != ISD::ADD ||
3934 !isa<ConstantSDNode>(Val: User->getOperand(Num: 1)) ||
3935 !isInt<12>(x: cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue()))
3936 return false;
3937
3938 FoundADDI = true;
3939
3940 // If we have a SHXADD instruction, prefer that over reassociating an ADDI.
3941 assert(Shift.getOpcode() == ISD::SHL);
3942 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
3943 if (Subtarget.hasShlAdd(ShAmt: ShiftAmt))
3944 return false;
3945
3946 // All users of the ADDI should be load/store.
3947 for (auto *ADDIUser : User->users())
3948 if (!isRegRegScaleLoadOrStore(User: ADDIUser, Add: SDValue(User, 0), Subtarget))
3949 return false;
3950 }
3951
3952 return true;
3953}
3954
3955bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
3956 ArrayRef<unsigned> Amounts,
3957 SDValue &Base, SDValue &Index,
3958 SDValue &Scale) {
3959 if (Addr.getOpcode() != ISD::ADD)
3960 return false;
3961 SDValue LHS = Addr.getOperand(i: 0);
3962 SDValue RHS = Addr.getOperand(i: 1);
3963
3964 EVT VT = Addr.getSimpleValueType();
3965 auto SelectShl = [this, VT, Amounts](SDValue N, SDValue &Index,
3966 SDValue &Shift) {
3967 if (N.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
3968 return false;
3969
3970 // Only match shifts by a value in range [0, MaxShiftAmount].
3971 unsigned ShiftAmt = N.getConstantOperandVal(i: 1);
3972 if (!llvm::is_contained(Range: Amounts, Element: ShiftAmt))
3973 return false;
3974
3975 Index = N.getOperand(i: 0);
3976 Shift = CurDAG->getTargetConstant(Val: ShiftAmt, DL: SDLoc(N), VT);
3977 return true;
3978 };
3979
3980 if (auto *C1 = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3981 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
3982 if (LHS.getOpcode() == ISD::ADD &&
3983 !isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
3984 isInt<12>(x: C1->getSExtValue())) {
3985 if (SelectShl(LHS.getOperand(i: 1), Index, Scale) &&
3986 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 1))) {
3987 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
3988 DL: SDLoc(Addr), VT);
3989 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
3990 Op1: LHS.getOperand(i: 0), Op2: C1Val),
3991 0);
3992 return true;
3993 }
3994
3995 // Add is commutative so we need to check both operands.
3996 if (SelectShl(LHS.getOperand(i: 0), Index, Scale) &&
3997 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 0))) {
3998 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
3999 DL: SDLoc(Addr), VT);
4000 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
4001 Op1: LHS.getOperand(i: 1), Op2: C1Val),
4002 0);
4003 return true;
4004 }
4005 }
4006
4007 // Don't match add with constants.
4008 // FIXME: Is this profitable for large constants that have 0s in the lower
4009 // 12 bits that we can materialize with LUI?
4010 return false;
4011 }
4012
4013 // Try to match a shift on the RHS.
4014 if (SelectShl(RHS, Index, Scale)) {
4015 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: RHS))
4016 return false;
4017 Base = LHS;
4018 return true;
4019 }
4020
4021 // Try to match a shift on the LHS.
4022 if (SelectShl(LHS, Index, Scale)) {
4023 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: LHS))
4024 return false;
4025 Base = RHS;
4026 return true;
4027 }
4028
4029 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr))
4030 return false;
4031
4032 // Bail out if 0 is not in candidate shift amounts.
4033 if (!llvm::is_contained(Range&: Amounts, Element: 0))
4034 return false;
4035
4036 Base = LHS;
4037 Index = RHS;
4038 Scale = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT);
4039 return true;
4040}
4041
4042bool RISCVDAGToDAGISel::SelectAddrRegZextRegScale(SDValue Addr,
4043 ArrayRef<unsigned> Amounts,
4044 unsigned Bits, SDValue &Base,
4045 SDValue &Index,
4046 SDValue &Scale) {
4047 if (!SelectAddrRegRegScale(Addr, Amounts, Base, Index, Scale))
4048 return false;
4049
4050 if (Index.getOpcode() == ISD::AND) {
4051 auto *C = dyn_cast<ConstantSDNode>(Val: Index.getOperand(i: 1));
4052 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4053 Index = Index.getOperand(i: 0);
4054 return true;
4055 }
4056 }
4057
4058 return false;
4059}
4060
4061bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base,
4062 SDValue &Offset) {
4063 if (Addr.getOpcode() != ISD::ADD)
4064 return false;
4065
4066 if (isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)))
4067 return false;
4068
4069 Base = Addr.getOperand(i: 0);
4070 Offset = Addr.getOperand(i: 1);
4071 return true;
4072}
4073
4074bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
4075 SDValue &ShAmt) {
4076 ShAmt = N;
4077
4078 // Peek through zext.
4079 if (ShAmt->getOpcode() == ISD::ZERO_EXTEND)
4080 ShAmt = ShAmt.getOperand(i: 0);
4081
4082 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
4083 // amount. If there is an AND on the shift amount, we can bypass it if it
4084 // doesn't affect any of those bits.
4085 if (ShAmt.getOpcode() == ISD::AND &&
4086 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4087 const APInt &AndMask = ShAmt.getConstantOperandAPInt(i: 1);
4088
4089 // Since the max shift amount is a power of 2 we can subtract 1 to make a
4090 // mask that covers the bits needed to represent all shift amounts.
4091 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
4092 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
4093
4094 if (ShMask.isSubsetOf(RHS: AndMask)) {
4095 ShAmt = ShAmt.getOperand(i: 0);
4096 } else {
4097 // SimplifyDemandedBits may have optimized the mask so try restoring any
4098 // bits that are known zero.
4099 KnownBits Known = CurDAG->computeKnownBits(Op: ShAmt.getOperand(i: 0));
4100 if (!ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
4101 return true;
4102 ShAmt = ShAmt.getOperand(i: 0);
4103 }
4104 }
4105
4106 if (ShAmt.getOpcode() == ISD::ADD &&
4107 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4108 uint64_t Imm = ShAmt.getConstantOperandVal(i: 1);
4109 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
4110 // to avoid the ADD.
4111 if (Imm != 0 && Imm % ShiftWidth == 0) {
4112 ShAmt = ShAmt.getOperand(i: 0);
4113 return true;
4114 }
4115 } else if (ShAmt.getOpcode() == ISD::SUB &&
4116 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 0))) {
4117 uint64_t Imm = ShAmt.getConstantOperandVal(i: 0);
4118 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
4119 // generate a NEG instead of a SUB of a constant.
4120 if (Imm != 0 && Imm % ShiftWidth == 0) {
4121 SDLoc DL(ShAmt);
4122 EVT VT = ShAmt.getValueType();
4123 SDValue Zero = CurDAG->getRegister(Reg: RISCV::X0, VT);
4124 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
4125 MachineSDNode *Neg = CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero,
4126 Op2: ShAmt.getOperand(i: 1));
4127 ShAmt = SDValue(Neg, 0);
4128 return true;
4129 }
4130 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4131 // to generate a NOT instead of a SUB of a constant.
4132 if (Imm % ShiftWidth == ShiftWidth - 1) {
4133 SDLoc DL(ShAmt);
4134 EVT VT = ShAmt.getValueType();
4135 MachineSDNode *Not = CurDAG->getMachineNode(
4136 Opcode: RISCV::XORI, dl: DL, VT, Op1: ShAmt.getOperand(i: 1),
4137 Op2: CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/IsTarget: true));
4138 ShAmt = SDValue(Not, 0);
4139 return true;
4140 }
4141 }
4142
4143 return true;
4144}
4145
4146/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
4147/// check for equality with 0. This function emits instructions that convert the
4148/// seteq/setne into something that can be compared with 0.
4149/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
4150/// ISD::SETNE).
4151bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
4152 SDValue &Val, bool OneUse) {
4153 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
4154 "Unexpected condition code!");
4155
4156 // We're looking for a setcc.
4157 if (N->getOpcode() != ISD::SETCC)
4158 return false;
4159
4160 if (OneUse && !N->hasOneUse())
4161 return false;
4162
4163 // Must be an equality comparison.
4164 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
4165 if (CCVal != ExpectedCCVal)
4166 return false;
4167
4168 SDValue LHS = N->getOperand(Num: 0);
4169 SDValue RHS = N->getOperand(Num: 1);
4170
4171 if (!LHS.getValueType().isScalarInteger())
4172 return false;
4173
4174 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
4175 if (isNullConstant(V: RHS)) {
4176 Val = LHS;
4177 return true;
4178 }
4179
4180 SDLoc DL(N);
4181
4182 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4183 int64_t CVal = C->getSExtValue();
4184 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
4185 // non-zero otherwise.
4186 if (CVal == -2048) {
4187 Val = SDValue(
4188 CurDAG->getMachineNode(
4189 Opcode: RISCV::XORI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4190 Op2: CurDAG->getSignedTargetConstant(Val: CVal, DL, VT: N->getValueType(ResNo: 0))),
4191 0);
4192 return true;
4193 }
4194 // If the RHS is [-2047,2048], we can use addi/addiw with -RHS to produce 0
4195 // if the LHS is equal to the RHS and non-zero otherwise.
4196 if (isInt<12>(x: CVal) || CVal == 2048) {
4197 unsigned Opc = RISCV::ADDI;
4198 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4199 cast<VTSDNode>(Val: LHS.getOperand(i: 1))->getVT() == MVT::i32) {
4200 Opc = RISCV::ADDIW;
4201 LHS = LHS.getOperand(i: 0);
4202 }
4203
4204 Val = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4205 Op2: CurDAG->getSignedTargetConstant(
4206 Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4207 0);
4208 return true;
4209 }
4210 if (isPowerOf2_64(Value: CVal) && Subtarget->hasStdExtZbs()) {
4211 Val = SDValue(
4212 CurDAG->getMachineNode(
4213 Opcode: RISCV::BINVI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4214 Op2: CurDAG->getTargetConstant(Val: Log2_64(Value: CVal), DL, VT: N->getValueType(ResNo: 0))),
4215 0);
4216 return true;
4217 }
4218 // Same as the addi case above but for larger immediates (signed 26-bit) use
4219 // the QC_E_ADDI instruction from the Xqcilia extension, if available. Avoid
4220 // anything which can be done with a single lui as it might be compressible.
4221 if (Subtarget->hasVendorXqcilia() && isInt<26>(x: CVal) &&
4222 (CVal & 0xFFF) != 0) {
4223 Val = SDValue(
4224 CurDAG->getMachineNode(
4225 Opcode: RISCV::QC_E_ADDI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4226 Op2: CurDAG->getSignedTargetConstant(Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4227 0);
4228 return true;
4229 }
4230 }
4231
4232 // If nothing else we can XOR the LHS and RHS to produce zero if they are
4233 // equal and a non-zero value if they aren't.
4234 Val = SDValue(
4235 CurDAG->getMachineNode(Opcode: RISCV::XOR, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS, Op2: RHS), 0);
4236 return true;
4237}
4238
4239bool RISCVDAGToDAGISel::selectSExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4240 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4241 cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT().getSizeInBits() == Bits) {
4242 Val = N.getOperand(i: 0);
4243 return true;
4244 }
4245
4246 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
4247 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
4248 return N;
4249
4250 SDValue N0 = N.getOperand(i: 0);
4251 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4252 N.getConstantOperandVal(i: 1) == ShiftAmt &&
4253 N0.getConstantOperandVal(i: 1) == ShiftAmt)
4254 return N0.getOperand(i: 0);
4255
4256 return N;
4257 };
4258
4259 MVT VT = N.getSimpleValueType();
4260 if (CurDAG->ComputeNumSignBits(Op: N) > (VT.getSizeInBits() - Bits)) {
4261 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
4262 return true;
4263 }
4264
4265 return false;
4266}
4267
4268bool RISCVDAGToDAGISel::selectZExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4269 if (N.getOpcode() == ISD::AND) {
4270 auto *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
4271 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4272 Val = N.getOperand(i: 0);
4273 return true;
4274 }
4275 }
4276 MVT VT = N.getSimpleValueType();
4277 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: Bits);
4278 if (CurDAG->MaskedValueIsZero(Op: N, Mask)) {
4279 Val = N;
4280 return true;
4281 }
4282
4283 return false;
4284}
4285
4286/// Look for various patterns that can be done with a SHL that can be folded
4287/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
4288/// SHXADD we are trying to match.
4289bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
4290 SDValue &Val) {
4291 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4292 SDValue N0 = N.getOperand(i: 0);
4293
4294 if (bool LeftShift = N0.getOpcode() == ISD::SHL;
4295 (LeftShift || N0.getOpcode() == ISD::SRL) &&
4296 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4297 uint64_t Mask = N.getConstantOperandVal(i: 1);
4298 unsigned C2 = N0.getConstantOperandVal(i: 1);
4299
4300 unsigned XLen = Subtarget->getXLen();
4301 if (LeftShift)
4302 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4303 else
4304 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
4305
4306 if (isShiftedMask_64(Value: Mask)) {
4307 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4308 unsigned Trailing = llvm::countr_zero(Val: Mask);
4309 if (Trailing != ShAmt)
4310 return false;
4311
4312 unsigned Opcode;
4313 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
4314 // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
4315 // followed by a SHXADD with c3 for the X amount.
4316 if (LeftShift && Leading == 0 && C2 < Trailing)
4317 Opcode = RISCV::SRLI;
4318 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
4319 // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
4320 // followed by a SHXADD with c3 for the X amount.
4321 else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
4322 Opcode = RISCV::SRLIW;
4323 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
4324 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
4325 // followed by a SHXADD using c3 for the X amount.
4326 else if (!LeftShift && Leading == C2)
4327 Opcode = RISCV::SRLI;
4328 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
4329 // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
4330 // followed by a SHXADD using c3 for the X amount.
4331 else if (!LeftShift && Leading == 32 + C2)
4332 Opcode = RISCV::SRLIW;
4333 else
4334 return false;
4335
4336 SDLoc DL(N);
4337 EVT VT = N.getValueType();
4338 ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
4339 Val = SDValue(
4340 CurDAG->getMachineNode(Opcode, dl: DL, VT, Op1: N0.getOperand(i: 0),
4341 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT)),
4342 0);
4343 return true;
4344 }
4345 } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
4346 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4347 uint64_t Mask = N.getConstantOperandVal(i: 1);
4348 unsigned C2 = N0.getConstantOperandVal(i: 1);
4349
4350 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
4351 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
4352 // use (srli (srai y, c2 - c3), c3 + c4) followed by a SHXADD with c4 as
4353 // the X amount.
4354 if (isShiftedMask_64(Value: Mask)) {
4355 unsigned XLen = Subtarget->getXLen();
4356 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4357 unsigned Trailing = llvm::countr_zero(Val: Mask);
4358 if (C2 > Leading && Leading > 0 && Trailing == ShAmt) {
4359 SDLoc DL(N);
4360 EVT VT = N.getValueType();
4361 Val = SDValue(CurDAG->getMachineNode(
4362 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4363 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT)),
4364 0);
4365 Val = SDValue(CurDAG->getMachineNode(
4366 Opcode: RISCV::SRLI, dl: DL, VT, Op1: Val,
4367 Op2: CurDAG->getTargetConstant(Val: Leading + ShAmt, DL, VT)),
4368 0);
4369 return true;
4370 }
4371 }
4372 }
4373 } else if (bool LeftShift = N.getOpcode() == ISD::SHL;
4374 (LeftShift || N.getOpcode() == ISD::SRL) &&
4375 isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4376 SDValue N0 = N.getOperand(i: 0);
4377 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
4378 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4379 uint64_t Mask = N0.getConstantOperandVal(i: 1);
4380 if (isShiftedMask_64(Value: Mask)) {
4381 unsigned C1 = N.getConstantOperandVal(i: 1);
4382 unsigned XLen = Subtarget->getXLen();
4383 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4384 unsigned Trailing = llvm::countr_zero(Val: Mask);
4385 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
4386 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
4387 if (LeftShift && Leading == 32 && Trailing > 0 &&
4388 (Trailing + C1) == ShAmt) {
4389 SDLoc DL(N);
4390 EVT VT = N.getValueType();
4391 Val = SDValue(CurDAG->getMachineNode(
4392 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4393 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4394 0);
4395 return true;
4396 }
4397 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
4398 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
4399 if (!LeftShift && Leading == 32 && Trailing > C1 &&
4400 (Trailing - C1) == ShAmt) {
4401 SDLoc DL(N);
4402 EVT VT = N.getValueType();
4403 Val = SDValue(CurDAG->getMachineNode(
4404 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4405 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4406 0);
4407 return true;
4408 }
4409 }
4410 }
4411 }
4412
4413 return false;
4414}
4415
4416/// Look for various patterns that can be done with a SHL that can be folded
4417/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
4418/// SHXADD_UW we are trying to match.
4419bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
4420 SDValue &Val) {
4421 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
4422 N.hasOneUse()) {
4423 SDValue N0 = N.getOperand(i: 0);
4424 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4425 N0.hasOneUse()) {
4426 uint64_t Mask = N.getConstantOperandVal(i: 1);
4427 unsigned C2 = N0.getConstantOperandVal(i: 1);
4428
4429 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4430
4431 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
4432 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
4433 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
4434 if (isShiftedMask_64(Value: Mask)) {
4435 unsigned Leading = llvm::countl_zero(Val: Mask);
4436 unsigned Trailing = llvm::countr_zero(Val: Mask);
4437 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
4438 SDLoc DL(N);
4439 EVT VT = N.getValueType();
4440 Val = SDValue(CurDAG->getMachineNode(
4441 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4442 Op2: CurDAG->getTargetConstant(Val: C2 - ShAmt, DL, VT)),
4443 0);
4444 return true;
4445 }
4446 }
4447 }
4448 }
4449
4450 return false;
4451}
4452
4453bool RISCVDAGToDAGISel::orDisjoint(const SDNode *N) const {
4454 assert(N->getOpcode() == ISD::OR || N->getOpcode() == RISCVISD::OR_VL);
4455 if (N->getFlags().hasDisjoint())
4456 return true;
4457 return CurDAG->haveNoCommonBitsSet(A: N->getOperand(Num: 0), B: N->getOperand(Num: 1));
4458}
4459
4460bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
4461 SDValue N, SDValue &Val) {
4462 int OrigCost = RISCVMatInt::getIntMatCost(Val: APInt(64, OrigImm), Size: 64, STI: *Subtarget,
4463 /*CompressionCost=*/true);
4464 int Cost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
4465 /*CompressionCost=*/true);
4466 if (OrigCost <= Cost)
4467 return false;
4468
4469 Val = selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm, Subtarget: *Subtarget);
4470 return true;
4471}
4472
4473bool RISCVDAGToDAGISel::selectZExtImm32(SDValue N, SDValue &Val) {
4474 if (!isa<ConstantSDNode>(Val: N))
4475 return false;
4476 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4477 if ((Imm >> 31) != 1)
4478 return false;
4479
4480 for (const SDNode *U : N->users()) {
4481 switch (U->getOpcode()) {
4482 case ISD::ADD:
4483 break;
4484 case ISD::OR:
4485 if (orDisjoint(N: U))
4486 break;
4487 return false;
4488 default:
4489 return false;
4490 }
4491 }
4492
4493 return selectImm64IfCheaper(Imm: 0xffffffff00000000 | Imm, OrigImm: Imm, N, Val);
4494}
4495
4496bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
4497 if (!isa<ConstantSDNode>(Val: N))
4498 return false;
4499 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4500 if (isInt<32>(x: Imm))
4501 return false;
4502 if (Imm == INT64_MIN)
4503 return false;
4504
4505 for (const SDNode *U : N->users()) {
4506 switch (U->getOpcode()) {
4507 case ISD::ADD:
4508 break;
4509 case RISCVISD::VMV_V_X_VL:
4510 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4511 return V->getOpcode() == ISD::ADD ||
4512 V->getOpcode() == RISCVISD::ADD_VL;
4513 }))
4514 return false;
4515 break;
4516 default:
4517 return false;
4518 }
4519 }
4520
4521 return selectImm64IfCheaper(Imm: -Imm, OrigImm: Imm, N, Val);
4522}
4523
4524bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
4525 if (!isa<ConstantSDNode>(Val: N))
4526 return false;
4527 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4528
4529 // For 32-bit signed constants, we can only substitute LUI+ADDI with LUI.
4530 if (isInt<32>(x: Imm) && ((Imm & 0xfff) != 0xfff || Imm == -1))
4531 return false;
4532
4533 // Abandon this transform if the constant is needed elsewhere.
4534 for (const SDNode *U : N->users()) {
4535 switch (U->getOpcode()) {
4536 case ISD::AND:
4537 case ISD::OR:
4538 case ISD::XOR:
4539 if (!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()))
4540 return false;
4541 break;
4542 case RISCVISD::VMV_V_X_VL:
4543 if (!Subtarget->hasStdExtZvkb())
4544 return false;
4545 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4546 return V->getOpcode() == ISD::AND ||
4547 V->getOpcode() == RISCVISD::AND_VL;
4548 }))
4549 return false;
4550 break;
4551 default:
4552 return false;
4553 }
4554 }
4555
4556 if (isInt<32>(x: Imm)) {
4557 Val =
4558 selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm: ~Imm, Subtarget: *Subtarget);
4559 return true;
4560 }
4561
4562 // For 64-bit constants, the instruction sequences get complex,
4563 // so we select inverted only if it's cheaper.
4564 return selectImm64IfCheaper(Imm: ~Imm, OrigImm: Imm, N, Val);
4565}
4566
4567static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
4568 unsigned Bits,
4569 const TargetInstrInfo *TII) {
4570 unsigned MCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: User->getMachineOpcode());
4571
4572 if (!MCOpcode)
4573 return false;
4574
4575 const MCInstrDesc &MCID = TII->get(Opcode: User->getMachineOpcode());
4576 const uint64_t TSFlags = MCID.TSFlags;
4577 if (!RISCVII::hasSEWOp(TSFlags))
4578 return false;
4579 assert(RISCVII::hasVLOp(TSFlags));
4580
4581 unsigned ChainOpIdx = User->getNumOperands() - 1;
4582 bool HasChainOp = User->getOperand(Num: ChainOpIdx).getValueType() == MVT::Other;
4583 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
4584 unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
4585 const unsigned Log2SEW = User->getConstantOperandVal(Num: VLIdx + 1);
4586
4587 if (UserOpNo == VLIdx)
4588 return false;
4589
4590 auto NumDemandedBits =
4591 RISCV::getVectorLowDemandedScalarBits(Opcode: MCOpcode, Log2SEW);
4592 return NumDemandedBits && Bits >= *NumDemandedBits;
4593}
4594
4595// Return true if all users of this SDNode* only consume the lower \p Bits.
4596// This can be used to form W instructions for add/sub/mul/shl even when the
4597// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
4598// SimplifyDemandedBits has made it so some users see a sext_inreg and some
4599// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
4600// the add/sub/mul/shl to become non-W instructions. By checking the users we
4601// may be able to use a W instruction and CSE with the other instruction if
4602// this has happened. We could try to detect that the CSE opportunity exists
4603// before doing this, but that would be more complicated.
4604bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits,
4605 const unsigned Depth) const {
4606 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
4607 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
4608 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
4609 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
4610 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
4611 isa<ConstantSDNode>(Node) || Depth != 0) &&
4612 "Unexpected opcode");
4613
4614 if (Depth >= SelectionDAG::MaxRecursionDepth)
4615 return false;
4616
4617 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
4618 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
4619 if (Depth == 0 && !Node->getValueType(ResNo: 0).isScalarInteger())
4620 return false;
4621
4622 for (SDUse &Use : Node->uses()) {
4623 SDNode *User = Use.getUser();
4624 // Users of this node should have already been instruction selected
4625 if (!User->isMachineOpcode())
4626 return false;
4627
4628 // TODO: Add more opcodes?
4629 switch (User->getMachineOpcode()) {
4630 default:
4631 if (vectorPseudoHasAllNBitUsers(User, UserOpNo: Use.getOperandNo(), Bits, TII))
4632 break;
4633 return false;
4634 case RISCV::ADDW:
4635 case RISCV::ADDIW:
4636 case RISCV::SUBW:
4637 case RISCV::MULW:
4638 case RISCV::SLLW:
4639 case RISCV::SLLIW:
4640 case RISCV::SRAW:
4641 case RISCV::SRAIW:
4642 case RISCV::SRLW:
4643 case RISCV::SRLIW:
4644 case RISCV::DIVW:
4645 case RISCV::DIVUW:
4646 case RISCV::REMW:
4647 case RISCV::REMUW:
4648 case RISCV::ROLW:
4649 case RISCV::RORW:
4650 case RISCV::RORIW:
4651 case RISCV::CLSW:
4652 case RISCV::CLZW:
4653 case RISCV::CTZW:
4654 case RISCV::CPOPW:
4655 case RISCV::SLLI_UW:
4656 case RISCV::ABSW:
4657 case RISCV::FMV_W_X:
4658 case RISCV::FCVT_H_W:
4659 case RISCV::FCVT_H_W_INX:
4660 case RISCV::FCVT_H_WU:
4661 case RISCV::FCVT_H_WU_INX:
4662 case RISCV::FCVT_S_W:
4663 case RISCV::FCVT_S_W_INX:
4664 case RISCV::FCVT_S_WU:
4665 case RISCV::FCVT_S_WU_INX:
4666 case RISCV::FCVT_D_W:
4667 case RISCV::FCVT_D_W_INX:
4668 case RISCV::FCVT_D_WU:
4669 case RISCV::FCVT_D_WU_INX:
4670 case RISCV::TH_REVW:
4671 case RISCV::TH_SRRIW:
4672 if (Bits >= 32)
4673 break;
4674 return false;
4675 case RISCV::SLL:
4676 case RISCV::SRA:
4677 case RISCV::SRL:
4678 case RISCV::ROL:
4679 case RISCV::ROR:
4680 case RISCV::BSET:
4681 case RISCV::BCLR:
4682 case RISCV::BINV:
4683 case RISCV::BEXT:
4684 // Shift amount and bit index operands only use log2(Xlen) bits.
4685 if (Use.getOperandNo() == 1 && Bits >= Log2_32(Value: Subtarget->getXLen()))
4686 break;
4687 return false;
4688 case RISCV::SLLI:
4689 // SLLI only uses the lower (XLen - ShAmt) bits.
4690 if (Bits >= Subtarget->getXLen() - User->getConstantOperandVal(Num: 1))
4691 break;
4692 return false;
4693 case RISCV::ANDI:
4694 if (Bits >= (unsigned)llvm::bit_width(Value: User->getConstantOperandVal(Num: 1)))
4695 break;
4696 goto RecCheck;
4697 case RISCV::ORI: {
4698 uint64_t Imm = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
4699 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(Value: ~Imm))
4700 break;
4701 [[fallthrough]];
4702 }
4703 case RISCV::AND:
4704 case RISCV::OR:
4705 case RISCV::XOR:
4706 case RISCV::XORI:
4707 case RISCV::ANDN:
4708 case RISCV::ORN:
4709 case RISCV::XNOR:
4710 case RISCV::SH1ADD:
4711 case RISCV::SH2ADD:
4712 case RISCV::SH3ADD:
4713 RecCheck:
4714 if (hasAllNBitUsers(Node: User, Bits, Depth: Depth + 1))
4715 break;
4716 return false;
4717 case RISCV::SRLI: {
4718 unsigned ShAmt = User->getConstantOperandVal(Num: 1);
4719 // If we are shifting right by less than Bits, and users don't demand any
4720 // bits that were shifted into [Bits-1:0], then we can consider this as an
4721 // N-Bit user.
4722 if (Bits > ShAmt && hasAllNBitUsers(Node: User, Bits: Bits - ShAmt, Depth: Depth + 1))
4723 break;
4724 return false;
4725 }
4726 case RISCV::SEXT_B:
4727 case RISCV::PACKH:
4728 if (Bits >= 8)
4729 break;
4730 return false;
4731 case RISCV::SEXT_H:
4732 case RISCV::FMV_H_X:
4733 case RISCV::ZEXT_H_RV32:
4734 case RISCV::ZEXT_H_RV64:
4735 case RISCV::PACKW:
4736 if (Bits >= 16)
4737 break;
4738 return false;
4739 case RISCV::PACK:
4740 if (Bits >= (Subtarget->getXLen() / 2))
4741 break;
4742 return false;
4743 case RISCV::PPAIRE_H:
4744 // If only the lower 32-bits of the result are used, then only the
4745 // lower 16 bits of the inputs are used.
4746 if (Bits >= 16 && hasAllNBitUsers(Node: User, Bits: 32, Depth: Depth + 1))
4747 break;
4748 return false;
4749 case RISCV::ADD_UW:
4750 case RISCV::SH1ADD_UW:
4751 case RISCV::SH2ADD_UW:
4752 case RISCV::SH3ADD_UW:
4753 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
4754 // 32 bits.
4755 if (Use.getOperandNo() == 0 && Bits >= 32)
4756 break;
4757 return false;
4758 case RISCV::SB:
4759 if (Use.getOperandNo() == 0 && Bits >= 8)
4760 break;
4761 return false;
4762 case RISCV::SH:
4763 if (Use.getOperandNo() == 0 && Bits >= 16)
4764 break;
4765 return false;
4766 case RISCV::SW:
4767 if (Use.getOperandNo() == 0 && Bits >= 32)
4768 break;
4769 return false;
4770 case RISCV::TH_EXT:
4771 case RISCV::TH_EXTU: {
4772 unsigned Msb = User->getConstantOperandVal(Num: 1);
4773 unsigned Lsb = User->getConstantOperandVal(Num: 2);
4774 // Behavior of Msb < Lsb is not well documented.
4775 if (Msb >= Lsb && Bits > Msb)
4776 break;
4777 return false;
4778 }
4779 }
4780 }
4781
4782 return true;
4783}
4784
4785// Select a constant that can be represented as (sign_extend(imm5) << imm2).
4786bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
4787 SDValue &Shl2) {
4788 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4789 if (!C)
4790 return false;
4791
4792 int64_t Offset = C->getSExtValue();
4793 for (unsigned Shift = 0; Shift < 4; Shift++) {
4794 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0)) {
4795 EVT VT = N->getValueType(ResNo: 0);
4796 Simm5 = CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(N), VT);
4797 Shl2 = CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(N), VT);
4798 return true;
4799 }
4800 }
4801
4802 return false;
4803}
4804
4805// Select VL as a 5 bit immediate or a value that will become a register. This
4806// allows us to choose between VSETIVLI or VSETVLI later.
4807bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
4808 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4809 if (C && isUInt<5>(x: C->getZExtValue())) {
4810 VL = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(N),
4811 VT: N->getValueType(ResNo: 0));
4812 } else if (C && C->isAllOnes()) {
4813 // Treat all ones as VLMax.
4814 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4815 VT: N->getValueType(ResNo: 0));
4816 } else if (isa<RegisterSDNode>(Val: N) &&
4817 cast<RegisterSDNode>(Val&: N)->getReg() == RISCV::X0) {
4818 // All our VL operands use an operand that allows GPRNoX0 or an immediate
4819 // as the register class. Convert X0 to a special immediate to pass the
4820 // MachineVerifier. This is recognized specially by the vsetvli insertion
4821 // pass.
4822 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4823 VT: N->getValueType(ResNo: 0));
4824 } else {
4825 VL = N;
4826 }
4827
4828 return true;
4829}
4830
4831static SDValue findVSplat(SDValue N) {
4832 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
4833 if (!N.getOperand(i: 0).isUndef())
4834 return SDValue();
4835 N = N.getOperand(i: 1);
4836 }
4837 SDValue Splat = N;
4838 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
4839 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
4840 !Splat.getOperand(i: 0).isUndef())
4841 return SDValue();
4842 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
4843 return Splat;
4844}
4845
4846bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
4847 SDValue Splat = findVSplat(N);
4848 if (!Splat)
4849 return false;
4850
4851 SplatVal = Splat.getOperand(i: 1);
4852 return true;
4853}
4854
4855static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
4856 SelectionDAG &DAG,
4857 const RISCVSubtarget &Subtarget,
4858 std::function<bool(int64_t)> ValidateImm,
4859 bool Decrement = false) {
4860 SDValue Splat = findVSplat(N);
4861 if (!Splat || !isa<ConstantSDNode>(Val: Splat.getOperand(i: 1)))
4862 return false;
4863
4864 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
4865 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
4866 "Unexpected splat operand type");
4867
4868 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
4869 // type is wider than the resulting vector element type: an implicit
4870 // truncation first takes place. Therefore, perform a manual
4871 // truncation/sign-extension in order to ignore any truncated bits and catch
4872 // any zero-extended immediate.
4873 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
4874 // sign-extending to (XLenVT -1).
4875 APInt SplatConst = Splat.getConstantOperandAPInt(i: 1).sextOrTrunc(width: SplatEltSize);
4876
4877 int64_t SplatImm = SplatConst.getSExtValue();
4878
4879 if (!ValidateImm(SplatImm))
4880 return false;
4881
4882 if (Decrement)
4883 SplatImm -= 1;
4884
4885 SplatVal =
4886 DAG.getSignedTargetConstant(Val: SplatImm, DL: SDLoc(N), VT: Subtarget.getXLenVT());
4887 return true;
4888}
4889
4890bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
4891 return selectVSplatImmHelper(N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4892 ValidateImm: [](int64_t Imm) { return isInt<5>(x: Imm); });
4893}
4894
4895bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
4896 return selectVSplatImmHelper(
4897 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4898 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4899 /*Decrement=*/true);
4900}
4901
4902bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal) {
4903 return selectVSplatImmHelper(
4904 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4905 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4906 /*Decrement=*/false);
4907}
4908
4909bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
4910 SDValue &SplatVal) {
4911 return selectVSplatImmHelper(
4912 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4913 ValidateImm: [](int64_t Imm) { return Imm != 0 && Imm >= -15 && Imm <= 16; },
4914 /*Decrement=*/true);
4915}
4916
4917bool RISCVDAGToDAGISel::selectVSplatUimm(SDValue N, unsigned Bits,
4918 SDValue &SplatVal) {
4919 return selectVSplatImmHelper(
4920 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4921 ValidateImm: [Bits](int64_t Imm) { return isUIntN(N: Bits, x: Imm); });
4922}
4923
4924bool RISCVDAGToDAGISel::selectVSplatImm64Neg(SDValue N, SDValue &SplatVal) {
4925 SDValue Splat = findVSplat(N);
4926 return Splat && selectNegImm(N: Splat.getOperand(i: 1), Val&: SplatVal);
4927}
4928
4929bool RISCVDAGToDAGISel::selectLow8BitsVSplat(SDValue N, SDValue &SplatVal) {
4930 auto IsExtOrTrunc = [](SDValue N) {
4931 switch (N->getOpcode()) {
4932 case ISD::SIGN_EXTEND:
4933 case ISD::ZERO_EXTEND:
4934 // There's no passthru on these _VL nodes so any VL/mask is ok, since any
4935 // inactive elements will be undef.
4936 case RISCVISD::TRUNCATE_VECTOR_VL:
4937 case RISCVISD::VSEXT_VL:
4938 case RISCVISD::VZEXT_VL:
4939 return true;
4940 default:
4941 return false;
4942 }
4943 };
4944
4945 // We can have multiple nested nodes, so unravel them all if needed.
4946 while (IsExtOrTrunc(N)) {
4947 if (!N.hasOneUse() || N.getScalarValueSizeInBits() < 8)
4948 return false;
4949 N = N->getOperand(Num: 0);
4950 }
4951
4952 return selectVSplat(N, SplatVal);
4953}
4954
4955bool RISCVDAGToDAGISel::selectScalarFPAsInt(SDValue N, SDValue &Imm) {
4956 // Allow bitcasts from XLenVT -> FP.
4957 if (N.getOpcode() == ISD::BITCAST &&
4958 N.getOperand(i: 0).getValueType() == Subtarget->getXLenVT()) {
4959 Imm = N.getOperand(i: 0);
4960 return true;
4961 }
4962 // Allow moves from XLenVT to FP.
4963 if (N.getOpcode() == RISCVISD::FMV_H_X ||
4964 N.getOpcode() == RISCVISD::FMV_W_X_RV64) {
4965 Imm = N.getOperand(i: 0);
4966 return true;
4967 }
4968
4969 // Otherwise, look for FP constants that can materialized with scalar int.
4970 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val: N.getNode());
4971 if (!CFP)
4972 return false;
4973 const APFloat &APF = CFP->getValueAPF();
4974 // td can handle +0.0 already.
4975 if (APF.isPosZero())
4976 return false;
4977
4978 MVT VT = CFP->getSimpleValueType(ResNo: 0);
4979
4980 MVT XLenVT = Subtarget->getXLenVT();
4981 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
4982 assert(APF.isNegZero() && "Unexpected constant.");
4983 return false;
4984 }
4985 SDLoc DL(N);
4986 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
4987 Subtarget: *Subtarget);
4988 return true;
4989}
4990
4991bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
4992 SDValue &Imm) {
4993 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N)) {
4994 int64_t ImmVal = SignExtend64(X: C->getSExtValue(), B: Width);
4995
4996 if (!isInt<5>(x: ImmVal))
4997 return false;
4998
4999 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N),
5000 VT: Subtarget->getXLenVT());
5001 return true;
5002 }
5003
5004 return false;
5005}
5006
5007// Match XOR with a VMSET_VL operand. Return the other operand.
5008bool RISCVDAGToDAGISel::selectVMNOTOp(SDValue N, SDValue &Res) {
5009 if (N.getOpcode() != ISD::XOR)
5010 return false;
5011
5012 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5013 Res = N.getOperand(i: 1);
5014 return true;
5015 }
5016
5017 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5018 Res = N.getOperand(i: 0);
5019 return true;
5020 }
5021
5022 return false;
5023}
5024
5025// Match VMXOR_VL with a VMSET_VL operand. Making sure that that VL operand
5026// matches the parent's VL. Return the other operand of the VMXOR_VL.
5027bool RISCVDAGToDAGISel::selectVMNOT_VLOp(SDNode *Parent, SDValue N,
5028 SDValue &Res) {
5029 if (N.getOpcode() != RISCVISD::VMXOR_VL)
5030 return false;
5031
5032 assert(Parent &&
5033 (Parent->getOpcode() == RISCVISD::VMAND_VL ||
5034 Parent->getOpcode() == RISCVISD::VMOR_VL ||
5035 Parent->getOpcode() == RISCVISD::VMXOR_VL) &&
5036 "Unexpected parent");
5037
5038 // The VL should match the parent.
5039 if (Parent->getOperand(Num: 2) != N->getOperand(Num: 2))
5040 return false;
5041
5042 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5043 Res = N.getOperand(i: 1);
5044 return true;
5045 }
5046
5047 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5048 Res = N.getOperand(i: 0);
5049 return true;
5050 }
5051
5052 return false;
5053}
5054
5055// Try to remove sext.w if the input is a W instruction or can be made into
5056// a W instruction cheaply.
5057bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
5058 // Look for the sext.w pattern, addiw rd, rs1, 0.
5059 if (N->getMachineOpcode() != RISCV::ADDIW ||
5060 !isNullConstant(V: N->getOperand(Num: 1)))
5061 return false;
5062
5063 SDValue N0 = N->getOperand(Num: 0);
5064 if (!N0.isMachineOpcode())
5065 return false;
5066
5067 switch (N0.getMachineOpcode()) {
5068 default:
5069 break;
5070 case RISCV::ADD:
5071 case RISCV::ADDI:
5072 case RISCV::SUB:
5073 case RISCV::MUL:
5074 case RISCV::SLLI: {
5075 // Convert sext.w+add/sub/mul to their W instructions. This will create
5076 // a new independent instruction. This improves latency.
5077 unsigned Opc;
5078 switch (N0.getMachineOpcode()) {
5079 default:
5080 llvm_unreachable("Unexpected opcode!");
5081 case RISCV::ADD: Opc = RISCV::ADDW; break;
5082 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
5083 case RISCV::SUB: Opc = RISCV::SUBW; break;
5084 case RISCV::MUL: Opc = RISCV::MULW; break;
5085 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
5086 }
5087
5088 SDValue N00 = N0.getOperand(i: 0);
5089 SDValue N01 = N0.getOperand(i: 1);
5090
5091 // Shift amount needs to be uimm5.
5092 if (N0.getMachineOpcode() == RISCV::SLLI &&
5093 !isUInt<5>(x: cast<ConstantSDNode>(Val&: N01)->getSExtValue()))
5094 break;
5095
5096 SDNode *Result =
5097 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VT: N->getValueType(ResNo: 0),
5098 Op1: N00, Op2: N01);
5099 ReplaceUses(F: N, T: Result);
5100 return true;
5101 }
5102 case RISCV::ADDW:
5103 case RISCV::ADDIW:
5104 case RISCV::SUBW:
5105 case RISCV::MULW:
5106 case RISCV::SLLIW:
5107 case RISCV::PACKW:
5108 case RISCV::TH_MULAW:
5109 case RISCV::TH_MULAH:
5110 case RISCV::TH_MULSW:
5111 case RISCV::TH_MULSH:
5112 if (N0.getValueType() == MVT::i32)
5113 break;
5114
5115 // Result is already sign extended just remove the sext.w.
5116 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
5117 ReplaceUses(F: N, T: N0.getNode());
5118 return true;
5119 }
5120
5121 return false;
5122}
5123
5124static bool usesAllOnesMask(SDValue MaskOp) {
5125 const auto IsVMSet = [](unsigned Opc) {
5126 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
5127 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
5128 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
5129 Opc == RISCV::PseudoVMSET_M_B8;
5130 };
5131
5132 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
5133 // undefined behaviour if it's the wrong bitwidth, so we could choose to
5134 // assume that it's all-ones? Same applies to its VL.
5135 return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode());
5136}
5137
5138static bool isImplicitDef(SDValue V) {
5139 if (!V.isMachineOpcode())
5140 return false;
5141 if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
5142 for (unsigned I = 1; I < V.getNumOperands(); I += 2)
5143 if (!isImplicitDef(V: V.getOperand(i: I)))
5144 return false;
5145 return true;
5146 }
5147 return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
5148}
5149
5150// Optimize masked RVV pseudo instructions with a known all-ones mask to their
5151// corresponding "unmasked" pseudo versions.
5152bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
5153 const RISCV::RISCVMaskedPseudoInfo *I =
5154 RISCV::getMaskedPseudoInfo(MaskedPseudo: N->getMachineOpcode());
5155 if (!I)
5156 return false;
5157
5158 unsigned MaskOpIdx = I->MaskOpIdx;
5159 if (!usesAllOnesMask(MaskOp: N->getOperand(Num: MaskOpIdx)))
5160 return false;
5161
5162 // There are two classes of pseudos in the table - compares and
5163 // everything else. See the comment on RISCVMaskedPseudo for details.
5164 const unsigned Opc = I->UnmaskedPseudo;
5165 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
5166 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MCID);
5167
5168 const MCInstrDesc &MaskedMCID = TII->get(Opcode: N->getMachineOpcode());
5169 const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MaskedMCID);
5170
5171 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
5172 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&
5173 "Unmasked pseudo has policy but masked pseudo doesn't?");
5174 assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru &&
5175 "Unexpected pseudo structure");
5176 assert(!(HasPassthru && !MaskedHasPassthru) &&
5177 "Unmasked pseudo has passthru but masked pseudo doesn't?");
5178
5179 SmallVector<SDValue, 8> Ops;
5180 // Skip the passthru operand at index 0 if the unmasked don't have one.
5181 bool ShouldSkip = !HasPassthru && MaskedHasPassthru;
5182 bool DropPolicy = !RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags) &&
5183 RISCVII::hasVecPolicyOp(TSFlags: MaskedMCID.TSFlags);
5184 bool HasChainOp =
5185 N->getOperand(Num: N->getNumOperands() - 1).getValueType() == MVT::Other;
5186 unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp;
5187 for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) {
5188 // Skip the mask
5189 SDValue Op = N->getOperand(Num: I);
5190 if (I == MaskOpIdx)
5191 continue;
5192 if (DropPolicy && I == LastOpNum)
5193 continue;
5194 Ops.push_back(Elt: Op);
5195 }
5196
5197 MachineSDNode *Result =
5198 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5199
5200 if (!N->memoperands_empty())
5201 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: N->memoperands());
5202
5203 Result->setFlags(N->getFlags());
5204 ReplaceUses(F: N, T: Result);
5205
5206 return true;
5207}
5208
5209/// If our passthru is an implicit_def, use noreg instead. This side
5210/// steps issues with MachineCSE not being able to CSE expressions with
5211/// IMPLICIT_DEF operands while preserving the semantic intent. See
5212/// pr64282 for context. Note that this transform is the last one
5213/// performed at ISEL DAG to DAG.
5214bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
5215 bool MadeChange = false;
5216 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5217
5218 while (Position != CurDAG->allnodes_begin()) {
5219 SDNode *N = &*--Position;
5220 if (N->use_empty() || !N->isMachineOpcode())
5221 continue;
5222
5223 const unsigned Opc = N->getMachineOpcode();
5224 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opc) ||
5225 !RISCVII::isFirstDefTiedToFirstUse(Desc: TII->get(Opcode: Opc)) ||
5226 !isImplicitDef(V: N->getOperand(Num: 0)))
5227 continue;
5228
5229 SmallVector<SDValue> Ops;
5230 Ops.push_back(Elt: CurDAG->getRegister(Reg: RISCV::NoRegister, VT: N->getValueType(ResNo: 0)));
5231 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
5232 SDValue Op = N->getOperand(Num: I);
5233 Ops.push_back(Elt: Op);
5234 }
5235
5236 MachineSDNode *Result =
5237 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5238 Result->setFlags(N->getFlags());
5239 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: cast<MachineSDNode>(Val: N)->memoperands());
5240 ReplaceUses(F: N, T: Result);
5241 MadeChange = true;
5242 }
5243 return MadeChange;
5244}
5245
5246
5247// This pass converts a legalized DAG into a RISCV-specific DAG, ready
5248// for instruction scheduling.
5249FunctionPass *llvm::createRISCVISelDagLegacyPass(RISCVTargetMachine &TM,
5250 CodeGenOptLevel OptLevel) {
5251 return new RISCVDAGToDAGISelLegacy(TM, OptLevel);
5252}
5253
5254RISCVISelDAGToDAGPass::RISCVISelDAGToDAGPass(RISCVTargetMachine &TM,
5255 CodeGenOptLevel OptLevel)
5256 : SelectionDAGISelPass(std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5257
5258char RISCVDAGToDAGISelLegacy::ID = 0;
5259
5260RISCVDAGToDAGISelLegacy::RISCVDAGToDAGISelLegacy(RISCVTargetMachine &TM,
5261 CodeGenOptLevel OptLevel)
5262 : SelectionDAGISelLegacy(
5263 ID, std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5264
5265INITIALIZE_PASS(RISCVDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
5266