1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMCTargetDesc.h"
16#include "MCTargetDesc/RISCVMatInt.h"
17#include "RISCVISelLowering.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSelectionDAGInfo.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/Support/Alignment.h"
23#include "llvm/Support/Debug.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
32#define GET_DAGISEL_BODY RISCVDAGToDAGISel
33#include "RISCVGenDAGISel.inc"
34
35void RISCVDAGToDAGISel::PreprocessISelDAG() {
36 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
37
38 bool MadeChange = false;
39 while (Position != CurDAG->allnodes_begin()) {
40 SDNode *N = &*--Position;
41 if (N->use_empty())
42 continue;
43
44 SDValue Result;
45 switch (N->getOpcode()) {
46 case ISD::SPLAT_VECTOR: {
47 if (Subtarget->hasStdExtP())
48 break;
49 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
50 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
51 MVT VT = N->getSimpleValueType(ResNo: 0);
52 unsigned Opc =
53 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
54 SDLoc DL(N);
55 SDValue VL = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
56 SDValue Src = N->getOperand(Num: 0);
57 if (VT.isInteger())
58 Src = CurDAG->getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget->getXLenVT(),
59 Operand: N->getOperand(Num: 0));
60 Result = CurDAG->getNode(Opcode: Opc, DL, VT, N1: CurDAG->getUNDEF(VT), N2: Src, N3: VL);
61 break;
62 }
63 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
64 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
65 // load. Done after lowering and combining so that we have a chance to
66 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
67 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
68 MVT VT = N->getSimpleValueType(ResNo: 0);
69 SDValue Passthru = N->getOperand(Num: 0);
70 SDValue Lo = N->getOperand(Num: 1);
71 SDValue Hi = N->getOperand(Num: 2);
72 SDValue VL = N->getOperand(Num: 3);
73 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
74 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
75 "Unexpected VTs!");
76 MachineFunction &MF = CurDAG->getMachineFunction();
77 SDLoc DL(N);
78
79 // Create temporary stack for each expanding node.
80 SDValue StackSlot =
81 CurDAG->CreateStackTemporary(Bytes: TypeSize::getFixed(ExactSize: 8), Alignment: Align(8));
82 int FI = cast<FrameIndexSDNode>(Val: StackSlot.getNode())->getIndex();
83 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
84
85 SDValue Chain = CurDAG->getEntryNode();
86 Lo = CurDAG->getStore(Chain, dl: DL, Val: Lo, Ptr: StackSlot, PtrInfo: MPI, Alignment: Align(8));
87
88 SDValue OffsetSlot =
89 CurDAG->getMemBasePlusOffset(Base: StackSlot, Offset: TypeSize::getFixed(ExactSize: 4), DL);
90 Hi = CurDAG->getStore(Chain, dl: DL, Val: Hi, Ptr: OffsetSlot, PtrInfo: MPI.getWithOffset(O: 4),
91 Alignment: Align(8));
92
93 Chain = CurDAG->getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo, N2: Hi);
94
95 SDVTList VTs = CurDAG->getVTList(VTs: {VT, MVT::Other});
96 SDValue IntID =
97 CurDAG->getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: MVT::i64);
98 SDValue Ops[] = {Chain,
99 IntID,
100 Passthru,
101 StackSlot,
102 CurDAG->getRegister(Reg: RISCV::X0, VT: MVT::i64),
103 VL};
104
105 Result = CurDAG->getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
106 MemVT: MVT::i64, PtrInfo: MPI, Alignment: Align(8),
107 Flags: MachineMemOperand::MOLoad);
108 break;
109 }
110 case ISD::FP_EXTEND: {
111 // We only have vector patterns for riscv_fpextend_vl in isel.
112 SDLoc DL(N);
113 MVT VT = N->getSimpleValueType(ResNo: 0);
114 if (!VT.isVector())
115 break;
116 SDValue VLMAX = CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT());
117 SDValue TrueMask = CurDAG->getNode(
118 Opcode: RISCVISD::VMSET_VL, DL, VT: VT.changeVectorElementType(EltVT: MVT::i1), Operand: VLMAX);
119 Result = CurDAG->getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT, N1: N->getOperand(Num: 0),
120 N2: TrueMask, N3: VLMAX);
121 break;
122 }
123 case ISD::ADD: {
124 // Turn (add X, C) into (sub X, -C) when a constant node holding -C
125 // already exists in the DAG, so both share one materialization. Do this
126 // before selection, while both are still ConstantSDNodes: by selection
127 // time -C may already have been selected into instructions.
128 //
129 // ADD is commutative, but getNode canonicalizes constants to the RHS, so
130 // the constant is always operand 1.
131 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
132 if (!N1C)
133 break;
134 MVT VT = N->getSimpleValueType(ResNo: 0);
135 if (VT != Subtarget->getXLenVT())
136 break;
137 int64_t Imm = N1C->getSExtValue();
138 // Only worthwhile for wide constants: values that fit in 32 bits take at
139 // most two instructions to materialize, matching the threshold used by
140 // selectNegImm. Skip INT64_MIN too, whose negation is itself.
141 if (isInt<32>(x: Imm) || Imm == INT64_MIN)
142 break;
143 // A constant is anchored if it has a user other than an ADD, i.e. it is
144 // materialized regardless of this fold. N1C is the (unique) node for Imm,
145 // so the positive side needs no search.
146 auto IsAnchored = [](const SDNode *C) {
147 return any_of(Range: C->users(), P: [](const SDNode *U) {
148 return U->getOpcode() != ISD::ADD;
149 });
150 };
151 // If Imm is materialized anyway, keep the ADD so it reuses Imm; an ADD is
152 // also more compressible than a SUB. This also lets us skip the search
153 // for -Imm below.
154 if (IsAnchored(N1C))
155 break;
156 // Find the (unique) constant node for -Imm, if any.
157 const SDNode *NegC = nullptr;
158 for (const SDNode &Node : CurDAG->allnodes()) {
159 auto *C = dyn_cast<ConstantSDNode>(Val: &Node);
160 if (C && C->getSimpleValueType(ResNo: 0) == VT && C->getSExtValue() == -Imm) {
161 NegC = &Node;
162 break;
163 }
164 }
165 // Reuse is only free if -Imm is already in the DAG.
166 if (!NegC)
167 break;
168 // dyn_cast<ConstantSDNode> also matches TargetConstant, which is encoded
169 // into the instruction rather than materialized, so reusing it would not
170 // remove a materialization. No TargetConstant is this wide (the largest
171 // are intrinsic IDs, which fit in 32 bits), so assert it is a Constant.
172 assert(NegC->getOpcode() == ISD::Constant &&
173 "Unexpected wide TargetConstant");
174 // Pick which of Imm/-Imm should be the surviving constant, so exactly
175 // one of the pair is materialized and any ADDs of the other reuse it:
176 // - if -Imm is materialized anyway, reuse it (rewrite to SUB);
177 // - else keep the cheaper constant, breaking ties towards the positive
178 // value so both ADDs of a C/-C pair agree on the survivor.
179 bool Rewrite;
180 if (IsAnchored(NegC)) {
181 Rewrite = true;
182 } else {
183 int PosCost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
184 /*CompressionCost=*/true);
185 int NegCost =
186 RISCVMatInt::getIntMatCost(Val: APInt(64, -Imm), Size: 64, STI: *Subtarget,
187 /*CompressionCost=*/true);
188 Rewrite = NegCost != PosCost ? NegCost < PosCost : Imm < 0;
189 }
190 if (!Rewrite)
191 break;
192 SDLoc DL(N);
193 // getConstant uniques onto the existing -C node, so it is shared.
194 Result = CurDAG->getNode(Opcode: ISD::SUB, DL, VT, N1: N->getOperand(Num: 0),
195 N2: CurDAG->getConstant(Val: -Imm, DL, VT));
196 break;
197 }
198 }
199
200 if (Result) {
201 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
202 LLVM_DEBUG(N->dump(CurDAG));
203 LLVM_DEBUG(dbgs() << "\nNew: ");
204 LLVM_DEBUG(Result->dump(CurDAG));
205 LLVM_DEBUG(dbgs() << "\n");
206
207 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
208 MadeChange = true;
209 }
210 }
211
212 if (MadeChange)
213 CurDAG->RemoveDeadNodes();
214}
215
216void RISCVDAGToDAGISel::PostprocessISelDAG() {
217 HandleSDNode Dummy(CurDAG->getRoot());
218 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
219
220 bool MadeChange = false;
221 while (Position != CurDAG->allnodes_begin()) {
222 SDNode *N = &*--Position;
223 // Skip dead nodes and any non-machine opcodes.
224 if (N->use_empty() || !N->isMachineOpcode())
225 continue;
226
227 MadeChange |= doPeepholeSExtW(Node: N);
228
229 // FIXME: This is here only because the VMerge transform doesn't
230 // know how to handle masked true inputs. Once that has been moved
231 // to post-ISEL, this can be deleted as well.
232 MadeChange |= doPeepholeMaskedRVV(Node: cast<MachineSDNode>(Val: N));
233 }
234
235 CurDAG->setRoot(Dummy.getValue());
236
237 // After we're done with everything else, convert IMPLICIT_DEF
238 // passthru operands to NoRegister. This is required to workaround
239 // an optimization deficiency in MachineCSE. This really should
240 // be merged back into each of the patterns (i.e. there's no good
241 // reason not to go directly to NoReg), but is being done this way
242 // to allow easy backporting.
243 MadeChange |= doPeepholeNoRegPassThru();
244
245 if (MadeChange)
246 CurDAG->RemoveDeadNodes();
247}
248
249static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
250 RISCVMatInt::InstSeq &Seq) {
251 SDValue SrcReg = CurDAG->getRegister(Reg: RISCV::X0, VT);
252 for (const RISCVMatInt::Inst &Inst : Seq) {
253 SDValue SDImm = CurDAG->getSignedTargetConstant(Val: Inst.getImm(), DL, VT);
254 SDNode *Result = nullptr;
255 switch (Inst.getOpndKind()) {
256 case RISCVMatInt::Imm:
257 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SDImm);
258 break;
259 case RISCVMatInt::RegX0:
260 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg,
261 Op2: CurDAG->getRegister(Reg: RISCV::X0, VT));
262 break;
263 case RISCVMatInt::RegReg:
264 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SrcReg);
265 break;
266 case RISCVMatInt::RegImm:
267 Result = CurDAG->getMachineNode(Opcode: Inst.getOpcode(), dl: DL, VT, Op1: SrcReg, Op2: SDImm);
268 break;
269 }
270
271 // Only the first instruction has X0 as its source.
272 SrcReg = SDValue(Result, 0);
273 }
274
275 return SrcReg;
276}
277
278static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
279 int64_t Imm, const RISCVSubtarget &Subtarget) {
280 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
281
282 // Use a rematerializable pseudo instruction for short sequences if enabled.
283 if (Seq.size() == 2 && Subtarget.getCLOpts().use_rematerializable_movimm)
284 return SDValue(
285 CurDAG->getMachineNode(Opcode: RISCV::PseudoMovImm, dl: DL, VT,
286 Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT)),
287 0);
288
289 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
290 // worst an LUI+ADDIW. This will require an extra register, but avoids a
291 // constant pool.
292 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
293 // low and high 32 bits are the same and bit 31 and 63 are set.
294 if (Seq.size() > 3) {
295 unsigned ShiftAmt, AddOpc;
296 RISCVMatInt::InstSeq SeqLo =
297 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
298 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
299 SDValue Lo = selectImmSeq(CurDAG, DL, VT, Seq&: SeqLo);
300
301 SDValue SLLI = SDValue(
302 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: Lo,
303 Op2: CurDAG->getTargetConstant(Val: ShiftAmt, DL, VT)),
304 0);
305 return SDValue(CurDAG->getMachineNode(Opcode: AddOpc, dl: DL, VT, Op1: Lo, Op2: SLLI), 0);
306 }
307 }
308
309 // Otherwise, use the original sequence.
310 return selectImmSeq(CurDAG, DL, VT, Seq);
311}
312
313void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
314 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
315 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
316 bool IsLoad, MVT *IndexVT) {
317 SDValue Chain = Node->getOperand(Num: 0);
318
319 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Base pointer.
320
321 if (IsStridedOrIndexed) {
322 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Index.
323 if (IndexVT)
324 *IndexVT = Operands.back()->getSimpleValueType(ResNo: 0);
325 }
326
327 if (IsMasked) {
328 SDValue Mask = Node->getOperand(Num: CurOp++);
329 Operands.push_back(Elt: Mask);
330 }
331 SDValue VL;
332 selectVLOp(N: Node->getOperand(Num: CurOp++), VL);
333 Operands.push_back(Elt: VL);
334
335 MVT XLenVT = Subtarget->getXLenVT();
336 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
337 Operands.push_back(Elt: SEWOp);
338
339 // At the IR layer, all the masked load intrinsics have policy operands,
340 // none of the others do. All have passthru operands. For our pseudos,
341 // all loads have policy operands.
342 if (IsLoad) {
343 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
344 if (IsMasked)
345 Policy = Node->getConstantOperandVal(Num: CurOp++);
346 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
347 Operands.push_back(Elt: PolicyOp);
348 }
349
350 Operands.push_back(Elt: Chain); // Chain.
351}
352
353void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked,
354 bool IsStrided) {
355 SDLoc DL(Node);
356 MVT VT = Node->getSimpleValueType(ResNo: 0);
357 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
358 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
359
360 unsigned CurOp = 2;
361 SmallVector<SDValue, 8> Operands;
362
363 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
364
365 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
366 Operands, /*IsLoad=*/true);
367
368 const RISCV::VLSEGPseudo *P =
369 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
370 LMUL: static_cast<unsigned>(LMUL));
371 MachineSDNode *Load =
372 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
373
374 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
375
376 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
377 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
378 CurDAG->RemoveDeadNode(N: Node);
379}
380
381void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, unsigned NF,
382 bool IsMasked) {
383 SDLoc DL(Node);
384 MVT VT = Node->getSimpleValueType(ResNo: 0);
385 MVT XLenVT = Subtarget->getXLenVT();
386 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
387 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
388
389 unsigned CurOp = 2;
390 SmallVector<SDValue, 7> Operands;
391
392 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
393
394 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
395 /*IsStridedOrIndexed*/ false, Operands,
396 /*IsLoad=*/true);
397
398 const RISCV::VLSEGPseudo *P =
399 RISCV::getVLSEGPseudo(NF, Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
400 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
401 MachineSDNode *Load = CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped,
402 VT2: XLenVT, VT3: MVT::Other, Ops: Operands);
403
404 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
405
406 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0)); // Result
407 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1)); // VL
408 ReplaceUses(F: SDValue(Node, 2), T: SDValue(Load, 2)); // Chain
409 CurDAG->RemoveDeadNode(N: Node);
410}
411
412void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked,
413 bool IsOrdered) {
414 SDLoc DL(Node);
415 MVT VT = Node->getSimpleValueType(ResNo: 0);
416 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
417 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
418
419 unsigned CurOp = 2;
420 SmallVector<SDValue, 8> Operands;
421
422 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
423
424 MVT IndexVT;
425 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
426 /*IsStridedOrIndexed*/ true, Operands,
427 /*IsLoad=*/true, IndexVT: &IndexVT);
428
429#ifndef NDEBUG
430 // Number of element = RVVBitsPerBlock * LMUL / SEW
431 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
432 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
433 if (DecodedLMUL.second)
434 ContainedTyNumElts /= DecodedLMUL.first;
435 else
436 ContainedTyNumElts *= DecodedLMUL.first;
437 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
438 "Element count mismatch");
439#endif
440
441 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
442 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
443 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
444 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
445 "values when XLEN=32");
446 }
447 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
448 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
449 IndexLMUL: static_cast<unsigned>(IndexLMUL));
450 MachineSDNode *Load =
451 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT1: MVT::Untyped, VT2: MVT::Other, Ops: Operands);
452
453 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
454
455 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Load, 0));
456 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Load, 1));
457 CurDAG->RemoveDeadNode(N: Node);
458}
459
460void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked,
461 bool IsStrided) {
462 SDLoc DL(Node);
463 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
464 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
465 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
466
467 unsigned CurOp = 2;
468 SmallVector<SDValue, 8> Operands;
469
470 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
471
472 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
473 Operands);
474
475 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
476 NF, Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
477 MachineSDNode *Store =
478 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
479
480 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
481
482 ReplaceNode(F: Node, T: Store);
483}
484
485void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked,
486 bool IsOrdered) {
487 SDLoc DL(Node);
488 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
489 unsigned Log2SEW = Node->getConstantOperandVal(Num: Node->getNumOperands() - 1);
490 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
491
492 unsigned CurOp = 2;
493 SmallVector<SDValue, 8> Operands;
494
495 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
496
497 MVT IndexVT;
498 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
499 /*IsStridedOrIndexed*/ true, Operands,
500 /*IsLoad=*/false, IndexVT: &IndexVT);
501
502#ifndef NDEBUG
503 // Number of element = RVVBitsPerBlock * LMUL / SEW
504 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
505 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
506 if (DecodedLMUL.second)
507 ContainedTyNumElts /= DecodedLMUL.first;
508 else
509 ContainedTyNumElts *= DecodedLMUL.first;
510 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
511 "Element count mismatch");
512#endif
513
514 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
515 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
516 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
517 reportFatalUsageError(reason: "The V extension does not support EEW=64 for index "
518 "values when XLEN=32");
519 }
520 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
521 NF, Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
522 IndexLMUL: static_cast<unsigned>(IndexLMUL));
523 MachineSDNode *Store =
524 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VT: Node->getValueType(ResNo: 0), Ops: Operands);
525
526 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
527
528 ReplaceNode(F: Node, T: Store);
529}
530
531void RISCVDAGToDAGISel::selectVSETVLI(SDNode *Node) {
532 if (!Subtarget->hasVInstructions())
533 return;
534
535 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
536
537 SDLoc DL(Node);
538 MVT XLenVT = Subtarget->getXLenVT();
539
540 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
541
542 assert((IntNo == Intrinsic::riscv_vsetvli ||
543 IntNo == Intrinsic::riscv_vsetvlimax) &&
544 "Unexpected vsetvli intrinsic");
545
546 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
547 unsigned Offset = (VLMax ? 1 : 2);
548
549 assert(Node->getNumOperands() == Offset + 2 &&
550 "Unexpected number of operands");
551
552 unsigned SEW =
553 RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: Offset) & 0x7);
554 RISCVVType::VLMUL VLMul = static_cast<RISCVVType::VLMUL>(
555 Node->getConstantOperandVal(Num: Offset + 1) & 0x7);
556
557 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL: VLMul, SEW, /*TailAgnostic*/ true,
558 /*MaskAgnostic*/ true);
559 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
560
561 SDValue VLOperand;
562 unsigned Opcode = RISCV::PseudoVSETVLI;
563 if (auto *C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1))) {
564 if (auto VLEN = Subtarget->getRealVLen())
565 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
566 VLMax = true;
567 }
568 if (VLMax || isAllOnesConstant(V: Node->getOperand(Num: 1))) {
569 VLOperand = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
570 Opcode = RISCV::PseudoVSETVLIX0;
571 } else {
572 VLOperand = Node->getOperand(Num: 1);
573
574 if (auto *C = dyn_cast<ConstantSDNode>(Val&: VLOperand)) {
575 uint64_t AVL = C->getZExtValue();
576 if (isUInt<5>(x: AVL)) {
577 SDValue VLImm = CurDAG->getTargetConstant(Val: AVL, DL, VT: XLenVT);
578 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: RISCV::PseudoVSETIVLI, dl: DL,
579 VT: XLenVT, Op1: VLImm, Op2: VTypeIOp));
580 return;
581 }
582 }
583 }
584
585 ReplaceNode(F: Node,
586 T: CurDAG->getMachineNode(Opcode, dl: DL, VT: XLenVT, Op1: VLOperand, Op2: VTypeIOp));
587}
588
589void RISCVDAGToDAGISel::selectXSfmmVSET(SDNode *Node) {
590 if (!Subtarget->hasVendorXSfmmbase())
591 return;
592
593 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
594
595 SDLoc DL(Node);
596 MVT XLenVT = Subtarget->getXLenVT();
597
598 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
599
600 assert((IntNo == Intrinsic::riscv_sf_vsettnt ||
601 IntNo == Intrinsic::riscv_sf_vsettm ||
602 IntNo == Intrinsic::riscv_sf_vsettk) &&
603 "Unexpected XSfmm vset intrinsic");
604
605 unsigned SEW = RISCVVType::decodeVSEW(VSEW: Node->getConstantOperandVal(Num: 2));
606 unsigned Widen = RISCVVType::decodeTWiden(TWiden: Node->getConstantOperandVal(Num: 3));
607 unsigned PseudoOpCode =
608 IntNo == Intrinsic::riscv_sf_vsettnt ? RISCV::PseudoSF_VSETTNT
609 : IntNo == Intrinsic::riscv_sf_vsettm ? RISCV::PseudoSF_VSETTM
610 : RISCV::PseudoSF_VSETTK;
611
612 if (IntNo == Intrinsic::riscv_sf_vsettnt) {
613 unsigned VTypeI = RISCVVType::encodeXSfmmVType(SEW, Widen, AltFmt: 0);
614 SDValue VTypeIOp = CurDAG->getTargetConstant(Val: VTypeI, DL, VT: XLenVT);
615
616 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
617 Op1: Node->getOperand(Num: 1), Op2: VTypeIOp));
618 } else {
619 SDValue Log2SEW = CurDAG->getTargetConstant(Val: Log2_32(Value: SEW), DL, VT: XLenVT);
620 SDValue TWiden = CurDAG->getTargetConstant(Val: Widen, DL, VT: XLenVT);
621 ReplaceNode(F: Node,
622 T: CurDAG->getMachineNode(Opcode: PseudoOpCode, dl: DL, VT: XLenVT,
623 Op1: Node->getOperand(Num: 1), Op2: Log2SEW, Op3: TWiden));
624 }
625}
626
627bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
628 MVT VT = Node->getSimpleValueType(ResNo: 0);
629 unsigned Opcode = Node->getOpcode();
630 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
631 "Unexpected opcode");
632 SDLoc DL(Node);
633
634 // For operations of the form (x << C1) op C2, check if we can use
635 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
636 SDValue N0 = Node->getOperand(Num: 0);
637 SDValue N1 = Node->getOperand(Num: 1);
638
639 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Val&: N1);
640 if (!Cst)
641 return false;
642
643 int64_t Val = Cst->getSExtValue();
644
645 // Check if immediate can already use ANDI/ORI/XORI.
646 if (isInt<12>(x: Val))
647 return false;
648
649 SDValue Shift = N0;
650
651 // If Val is simm32 and we have a sext_inreg from i32, then the binop
652 // produces at least 33 sign bits. We can peek through the sext_inreg and use
653 // a SLLIW at the end.
654 bool SignExt = false;
655 if (isInt<32>(x: Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
656 N0.hasOneUse() && cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT() == MVT::i32) {
657 SignExt = true;
658 Shift = N0.getOperand(i: 0);
659 }
660
661 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
662 return false;
663
664 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Val: Shift.getOperand(i: 1));
665 if (!ShlCst)
666 return false;
667
668 uint64_t ShAmt = ShlCst->getZExtValue();
669 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
670 "Unexpected shift amount!");
671
672 // Make sure that we don't change the operation by removing bits.
673 // This only matters for OR and XOR, AND is unaffected.
674 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(N: ShAmt);
675 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
676 return false;
677
678 int64_t ShiftedVal = Val >> ShAmt;
679 if (!isInt<12>(x: ShiftedVal))
680 return false;
681
682 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
683 if (SignExt && ShAmt >= 32)
684 return false;
685
686 // Ok, we can reorder to get a smaller immediate.
687 unsigned BinOpc;
688 switch (Opcode) {
689 default: llvm_unreachable("Unexpected opcode");
690 case ISD::AND: BinOpc = RISCV::ANDI; break;
691 case ISD::OR: BinOpc = RISCV::ORI; break;
692 case ISD::XOR: BinOpc = RISCV::XORI; break;
693 }
694
695 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
696
697 SDNode *BinOp = CurDAG->getMachineNode(
698 Opcode: BinOpc, dl: DL, VT, Op1: Shift.getOperand(i: 0),
699 Op2: CurDAG->getSignedTargetConstant(Val: ShiftedVal, DL, VT));
700 SDNode *SLLI =
701 CurDAG->getMachineNode(Opcode: ShOpc, dl: DL, VT, Op1: SDValue(BinOp, 0),
702 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
703 ReplaceNode(F: Node, T: SLLI);
704 return true;
705}
706
707bool RISCVDAGToDAGISel::trySignedBitfieldExtract(SDNode *Node) {
708 unsigned Opc;
709
710 if (Subtarget->hasVendorXTHeadBb())
711 Opc = RISCV::TH_EXT;
712 else if (Subtarget->hasVendorXAndesPerf())
713 Opc = RISCV::NDS_BFOS;
714 else if (Subtarget->hasVendorXqcibm())
715 Opc = RISCV::QC_EXT;
716 else
717 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
718 return false;
719
720 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
721 if (!N1C)
722 return false;
723
724 SDValue N0 = Node->getOperand(Num: 0);
725 if (!N0.hasOneUse())
726 return false;
727
728 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
729 const SDLoc &DL, MVT VT) {
730 if (Opc == RISCV::QC_EXT) {
731 // QC.EXT X, width, shamt
732 // shamt is the same as Lsb
733 // width is the number of bits to extract from the Lsb
734 Msb = Msb - Lsb + 1;
735 }
736 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
737 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
738 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
739 };
740
741 SDLoc DL(Node);
742 MVT VT = Node->getSimpleValueType(ResNo: 0);
743 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
744 const unsigned RightShAmt = N1C->getZExtValue();
745 assert(RightShAmt > 0 && RightShAmt < VT.getSizeInBits() &&
746 "Unexpected shift amount!");
747
748 // Transform (sra (shl X, C1) C2) with C1 < C2
749 // -> (SignedBitfieldExtract X, msb, lsb)
750 if (N0.getOpcode() == ISD::SHL) {
751 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
752 if (!N01C)
753 return false;
754
755 const unsigned LeftShAmt = N01C->getZExtValue();
756 assert(LeftShAmt > 0 && LeftShAmt < VT.getSizeInBits() &&
757 "Unexpected shift amount!");
758 // Make sure that this is a bitfield extraction (i.e., the shift-right
759 // amount can not be less than the left-shift).
760 if (LeftShAmt > RightShAmt)
761 return false;
762
763 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
764 const unsigned Msb = MsbPlusOne - 1;
765 const unsigned Lsb = RightShAmt - LeftShAmt;
766
767 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
768 ReplaceNode(F: Node, T: Sbe);
769 return true;
770 }
771
772 // Transform (sra (sext_inreg X, _), C) ->
773 // (SignedBitfieldExtract X, msb, lsb)
774 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
775 unsigned ExtSize =
776 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
777
778 // ExtSize of 32 should use sraiw via tablegen pattern.
779 if (ExtSize == 32)
780 return false;
781
782 const unsigned Msb = ExtSize - 1;
783 // If the shift-right amount is greater than Msb, it means that extracts
784 // the X[Msb] bit and sign-extend it.
785 const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
786
787 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
788 ReplaceNode(F: Node, T: Sbe);
789 return true;
790 }
791
792 return false;
793}
794
795bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
796 // Only supported with XAndesPerf at the moment.
797 if (!Subtarget->hasVendorXAndesPerf())
798 return false;
799
800 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
801 if (!N1C)
802 return false;
803
804 SDValue N0 = Node->getOperand(Num: 0);
805 if (!N0.hasOneUse())
806 return false;
807
808 auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
809 const SDLoc &DL, MVT VT) {
810 unsigned Opc = RISCV::NDS_BFOS;
811 // If the Lsb is equal to the Msb, then the Lsb should be 0.
812 if (Lsb == Msb)
813 Lsb = 0;
814 return CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: N0.getOperand(i: 0),
815 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
816 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
817 };
818
819 SDLoc DL(Node);
820 MVT VT = Node->getSimpleValueType(ResNo: 0);
821 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
822 const unsigned RightShAmt = N1C->getZExtValue();
823 assert(RightShAmt > 0 && RightShAmt < VT.getSizeInBits() &&
824 "Unexpected shift amount!");
825
826 // Transform (sra (shl X, C1) C2) with C1 > C2
827 // -> (NDS.BFOS X, lsb, msb)
828 if (N0.getOpcode() == ISD::SHL) {
829 auto *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
830 if (!N01C)
831 return false;
832
833 const unsigned LeftShAmt = N01C->getZExtValue();
834 assert(LeftShAmt > 0 && LeftShAmt < VT.getSizeInBits() &&
835 "Unexpected shift amount!");
836 // Make sure that this is a bitfield insertion (i.e., the shift-right
837 // amount should be less than the left-shift).
838 if (LeftShAmt <= RightShAmt)
839 return false;
840
841 const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
842 const unsigned Msb = MsbPlusOne - 1;
843 const unsigned Lsb = LeftShAmt - RightShAmt;
844
845 SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
846 ReplaceNode(F: Node, T: Sbi);
847 return true;
848 }
849
850 return false;
851}
852
853bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
854 const SDLoc &DL, MVT VT,
855 SDValue X, unsigned Msb,
856 unsigned Lsb) {
857 unsigned Opc;
858
859 if (Subtarget->hasVendorXTHeadBb()) {
860 Opc = RISCV::TH_EXTU;
861 } else if (Subtarget->hasVendorXAndesPerf()) {
862 Opc = RISCV::NDS_BFOZ;
863 } else if (Subtarget->hasVendorXqcibm()) {
864 Opc = RISCV::QC_EXTU;
865 // QC.EXTU X, width, shamt
866 // shamt is the same as Lsb
867 // width is the number of bits to extract from the Lsb
868 Msb = Msb - Lsb + 1;
869 } else {
870 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
871 return false;
872 }
873
874 SDNode *Ube = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
875 Op2: CurDAG->getTargetConstant(Val: Msb, DL, VT),
876 Op3: CurDAG->getTargetConstant(Val: Lsb, DL, VT));
877 ReplaceNode(F: Node, T: Ube);
878 return true;
879}
880
881bool RISCVDAGToDAGISel::tryUnsignedBitfieldInsertInZero(SDNode *Node,
882 const SDLoc &DL, MVT VT,
883 SDValue X, unsigned Msb,
884 unsigned Lsb) {
885 // Only supported with XAndesPerf at the moment.
886 if (!Subtarget->hasVendorXAndesPerf())
887 return false;
888
889 unsigned Opc = RISCV::NDS_BFOZ;
890
891 // If the Lsb is equal to the Msb, then the Lsb should be 0.
892 if (Lsb == Msb)
893 Lsb = 0;
894 SDNode *Ubi = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: X,
895 Op2: CurDAG->getTargetConstant(Val: Lsb, DL, VT),
896 Op3: CurDAG->getTargetConstant(Val: Msb, DL, VT));
897 ReplaceNode(F: Node, T: Ubi);
898 return true;
899}
900
901bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
902 // Target does not support indexed loads.
903 if (!Subtarget->hasVendorXTHeadMemIdx())
904 return false;
905
906 LoadSDNode *Ld = cast<LoadSDNode>(Val: Node);
907 ISD::MemIndexedMode AM = Ld->getAddressingMode();
908 if (AM == ISD::UNINDEXED)
909 return false;
910
911 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Ld->getOffset());
912 if (!C)
913 return false;
914
915 EVT LoadVT = Ld->getMemoryVT();
916 assert((AM == ISD::PRE_INC || AM == ISD::POST_INC) &&
917 "Unexpected addressing mode");
918 bool IsPre = AM == ISD::PRE_INC;
919 bool IsPost = AM == ISD::POST_INC;
920 int64_t Offset = C->getSExtValue();
921
922 // The constants that can be encoded in the THeadMemIdx instructions
923 // are of the form (sign_extend(imm5) << imm2).
924 unsigned Shift;
925 for (Shift = 0; Shift < 4; Shift++)
926 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
927 break;
928
929 // Constant cannot be encoded.
930 if (Shift == 4)
931 return false;
932
933 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
934 unsigned Opcode;
935 if (LoadVT == MVT::i8 && IsPre)
936 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
937 else if (LoadVT == MVT::i8 && IsPost)
938 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
939 else if (LoadVT == MVT::i16 && IsPre)
940 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
941 else if (LoadVT == MVT::i16 && IsPost)
942 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
943 else if (LoadVT == MVT::i32 && IsPre)
944 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
945 else if (LoadVT == MVT::i32 && IsPost)
946 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
947 else if (LoadVT == MVT::i64 && IsPre)
948 Opcode = RISCV::TH_LDIB;
949 else if (LoadVT == MVT::i64 && IsPost)
950 Opcode = RISCV::TH_LDIA;
951 else
952 return false;
953
954 EVT Ty = Ld->getOffset().getValueType();
955 SDValue Ops[] = {
956 Ld->getBasePtr(),
957 CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(Node), VT: Ty),
958 CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(Node), VT: Ty), Ld->getChain()};
959 SDNode *New = CurDAG->getMachineNode(Opcode, dl: SDLoc(Node), VT1: Ld->getValueType(ResNo: 0),
960 VT2: Ld->getValueType(ResNo: 1), VT3: MVT::Other, Ops);
961
962 MachineMemOperand *MemOp = cast<MemSDNode>(Val: Node)->getMemOperand();
963 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: New), NewMemRefs: {MemOp});
964
965 ReplaceNode(F: Node, T: New);
966
967 return true;
968}
969
970static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT,
971 SDValue Lo, SDValue Hi) {
972 SDValue Ops[] = {
973 CurDAG->getTargetConstant(Val: RISCV::GPRPairNoX0RegClassID, DL, VT: MVT::i32), Lo,
974 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_even, DL, VT: MVT::i32), Hi,
975 CurDAG->getTargetConstant(Val: RISCV::sub_gpr_odd, DL, VT: MVT::i32)};
976
977 return SDValue(
978 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT, Ops), 0);
979}
980
981// Helper to extract Lo and Hi values from a GPR pair.
982static std::pair<SDValue, SDValue>
983extractGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, SDValue Pair) {
984 SDValue Lo =
985 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL, VT: MVT::i32, Operand: Pair);
986 SDValue Hi =
987 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_odd, DL, VT: MVT::i32, Operand: Pair);
988 return {Lo, Hi};
989}
990
991// Try to match WMACC pattern: ADDD where one operand pair comes from a
992// widening multiply (both results of UMUL_LOHI, SMUL_LOHI, or WMULSU).
993bool RISCVDAGToDAGISel::tryWideningMulAcc(SDNode *Node, const SDLoc &DL) {
994 assert(Node->getOpcode() == RISCVISD::ADDD && "Expected ADDD");
995
996 SDValue Op0Lo = Node->getOperand(Num: 0);
997 SDValue Op0Hi = Node->getOperand(Num: 1);
998 SDValue Op1Lo = Node->getOperand(Num: 2);
999 SDValue Op1Hi = Node->getOperand(Num: 3);
1000
1001 auto IsSupportedMulWithOneUse = [](SDValue Lo, SDValue Hi) {
1002 unsigned Opc = Lo.getOpcode();
1003 if (Opc != ISD::UMUL_LOHI && Opc != ISD::SMUL_LOHI &&
1004 Opc != RISCVISD::WMULSU)
1005 return false;
1006 return Lo.getNode() == Hi.getNode() && Lo.getResNo() == 0 &&
1007 Hi.getResNo() == 1 && Lo.hasOneUse() && Hi.hasOneUse();
1008 };
1009
1010 SDNode *MulNode = nullptr;
1011 SDValue AddLo, AddHi;
1012
1013 // Check if first operand pair is a supported multiply with single use.
1014 if (IsSupportedMulWithOneUse(Op0Lo, Op0Hi)) {
1015 MulNode = Op0Lo.getNode();
1016 AddLo = Op1Lo;
1017 AddHi = Op1Hi;
1018 }
1019 // ADDD is commutative. Check if second operand pair is a supported multiply
1020 // with single use.
1021 else if (IsSupportedMulWithOneUse(Op1Lo, Op1Hi)) {
1022 MulNode = Op1Lo.getNode();
1023 AddLo = Op0Lo;
1024 AddHi = Op0Hi;
1025 } else {
1026 return false;
1027 }
1028
1029 unsigned Opc;
1030 switch (MulNode->getOpcode()) {
1031 default:
1032 llvm_unreachable("Unexpected multiply opcode");
1033 case ISD::UMUL_LOHI:
1034 Opc = RISCV::WMACCU;
1035 break;
1036 case ISD::SMUL_LOHI:
1037 Opc = RISCV::WMACC;
1038 break;
1039 case RISCVISD::WMULSU:
1040 Opc = RISCV::WMACCSU;
1041 break;
1042 }
1043
1044 SDValue Acc = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: AddLo, Hi: AddHi);
1045
1046 // WMACC instruction format: rd, rs1, rs2 (rd is accumulator).
1047 SDValue M0 = MulNode->getOperand(Num: 0);
1048 SDValue M1 = MulNode->getOperand(Num: 1);
1049 MachineSDNode *New =
1050 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Acc, Op2: M0, Op3: M1);
1051
1052 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
1053 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1054 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1055 CurDAG->RemoveDeadNode(N: Node);
1056 return true;
1057}
1058
1059static Register getTileReg(uint64_t TileNum) {
1060 assert(TileNum <= 15 && "Invalid tile number");
1061 return RISCV::T0 + TileNum;
1062}
1063
1064void RISCVDAGToDAGISel::selectSF_VC_X_SE(SDNode *Node) {
1065 if (!Subtarget->hasVInstructions())
1066 return;
1067
1068 assert(Node->getOpcode() == ISD::INTRINSIC_VOID && "Unexpected opcode");
1069
1070 SDLoc DL(Node);
1071 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
1072
1073 assert((IntNo == Intrinsic::riscv_sf_vc_x_se ||
1074 IntNo == Intrinsic::riscv_sf_vc_i_se) &&
1075 "Unexpected vsetvli intrinsic");
1076
1077 // imm, imm, imm, simm5/scalar, sew, log2lmul, vl
1078 unsigned Log2SEW = Log2_32(Value: Node->getConstantOperandVal(Num: 6));
1079 SDValue SEWOp =
1080 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: Subtarget->getXLenVT());
1081 SmallVector<SDValue, 8> Operands = {Node->getOperand(Num: 2), Node->getOperand(Num: 3),
1082 Node->getOperand(Num: 4), Node->getOperand(Num: 5),
1083 Node->getOperand(Num: 8), SEWOp,
1084 Node->getOperand(Num: 0)};
1085
1086 unsigned Opcode;
1087 auto *LMulSDNode = cast<ConstantSDNode>(Val: Node->getOperand(Num: 7));
1088 switch (LMulSDNode->getSExtValue()) {
1089 case 5:
1090 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
1091 : RISCV::PseudoSF_VC_I_SE_MF8;
1092 break;
1093 case 6:
1094 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
1095 : RISCV::PseudoSF_VC_I_SE_MF4;
1096 break;
1097 case 7:
1098 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
1099 : RISCV::PseudoSF_VC_I_SE_MF2;
1100 break;
1101 case 0:
1102 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
1103 : RISCV::PseudoSF_VC_I_SE_M1;
1104 break;
1105 case 1:
1106 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
1107 : RISCV::PseudoSF_VC_I_SE_M2;
1108 break;
1109 case 2:
1110 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
1111 : RISCV::PseudoSF_VC_I_SE_M4;
1112 break;
1113 case 3:
1114 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
1115 : RISCV::PseudoSF_VC_I_SE_M8;
1116 break;
1117 }
1118
1119 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
1120 Opcode, dl: DL, VT: Node->getSimpleValueType(ResNo: 0), Ops: Operands));
1121}
1122
1123static unsigned getSegInstNF(unsigned Intrinsic) {
1124#define INST_NF_CASE(NAME, NF) \
1125 case Intrinsic::riscv_##NAME##NF: \
1126 return NF;
1127#define INST_NF_CASE_MASK(NAME, NF) \
1128 case Intrinsic::riscv_##NAME##NF##_mask: \
1129 return NF;
1130#define INST_NF_CASE_FF(NAME, NF) \
1131 case Intrinsic::riscv_##NAME##NF##ff: \
1132 return NF;
1133#define INST_NF_CASE_FF_MASK(NAME, NF) \
1134 case Intrinsic::riscv_##NAME##NF##ff_mask: \
1135 return NF;
1136#define INST_ALL_NF_CASE_BASE(MACRO_NAME, NAME) \
1137 MACRO_NAME(NAME, 2) \
1138 MACRO_NAME(NAME, 3) \
1139 MACRO_NAME(NAME, 4) \
1140 MACRO_NAME(NAME, 5) \
1141 MACRO_NAME(NAME, 6) \
1142 MACRO_NAME(NAME, 7) \
1143 MACRO_NAME(NAME, 8)
1144#define INST_ALL_NF_CASE(NAME) \
1145 INST_ALL_NF_CASE_BASE(INST_NF_CASE, NAME) \
1146 INST_ALL_NF_CASE_BASE(INST_NF_CASE_MASK, NAME)
1147#define INST_ALL_NF_CASE_WITH_FF(NAME) \
1148 INST_ALL_NF_CASE(NAME) \
1149 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF, NAME) \
1150 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF_MASK, NAME)
1151 switch (Intrinsic) {
1152 default:
1153 llvm_unreachable("Unexpected segment load/store intrinsic");
1154 INST_ALL_NF_CASE_WITH_FF(vlseg)
1155 INST_ALL_NF_CASE(vlsseg)
1156 INST_ALL_NF_CASE(vloxseg)
1157 INST_ALL_NF_CASE(vluxseg)
1158 INST_ALL_NF_CASE(vsseg)
1159 INST_ALL_NF_CASE(vssseg)
1160 INST_ALL_NF_CASE(vsoxseg)
1161 INST_ALL_NF_CASE(vsuxseg)
1162 }
1163}
1164
1165static bool isApplicableToPLIOrPLUI(int Val) {
1166 // Check if the immediate is packed i8 or i10
1167 int16_t Bit31To16 = Val >> 16;
1168 int16_t Bit15To0 = Val;
1169 int8_t Bit15To8 = Bit15To0 >> 8;
1170 int8_t Bit7To0 = Val;
1171 if (Bit31To16 != Bit15To0)
1172 return false;
1173
1174 return isInt<10>(x: Bit15To0) || isShiftedInt<10, 6>(x: Bit15To0) ||
1175 Bit15To8 == Bit7To0;
1176}
1177
1178void RISCVDAGToDAGISel::Select(SDNode *Node) {
1179 // If we have a custom node, we have already selected.
1180 if (Node->isMachineOpcode()) {
1181 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
1182 Node->setNodeId(-1);
1183 return;
1184 }
1185
1186 // Instruction Selection not handled by the auto-generated tablegen selection
1187 // should be handled here.
1188 unsigned Opcode = Node->getOpcode();
1189 MVT XLenVT = Subtarget->getXLenVT();
1190 SDLoc DL(Node);
1191 MVT VT = Node->getSimpleValueType(ResNo: 0);
1192
1193 bool HasBitTest = Subtarget->hasBEXTILike();
1194
1195 switch (Opcode) {
1196 case ISD::Constant: {
1197 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1198 auto *ConstNode = cast<ConstantSDNode>(Val: Node);
1199 if (ConstNode->isZero()) {
1200 SDValue New =
1201 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: RISCV::X0, VT);
1202 ReplaceNode(F: Node, T: New.getNode());
1203 return;
1204 }
1205 int64_t Imm = ConstNode->getSExtValue();
1206 // If only the lower 8 bits are used, try to convert this to a simm6 by
1207 // sign-extending bit 7. This is neutral without the C extension, and
1208 // allows C.LI to be used if C is present.
1209 if (!isInt<8>(x: Imm) && isUInt<8>(x: Imm) && isInt<6>(x: SignExtend64<8>(x: Imm)) &&
1210 hasAllBUsers(Node))
1211 Imm = SignExtend64<8>(x: Imm);
1212 // If the upper XLen-16 bits are not used, try to convert this to a simm12
1213 // by sign extending bit 15.
1214 else if (!isInt<16>(x: Imm) && isUInt<16>(x: Imm) &&
1215 isInt<12>(x: SignExtend64<16>(x: Imm)) && hasAllHUsers(Node))
1216 Imm = SignExtend64<16>(x: Imm);
1217
1218 // If the upper XLen-16 bits are not used, the lower 2 bytes are the same,
1219 // and we can't use li, convert to an xlen splat so we can use pli.b.
1220 if (Subtarget->hasStdExtP() && !isInt<12>(x: Imm) &&
1221 (Imm & 0xff) == ((Imm >> 8) & 0xff) && hasAllHUsers(Node)) {
1222 // Splat the lower 16 bits to XLen. Sign extend for RV32.
1223 uint64_t Splat = Imm & 0xffff;
1224 Splat = (Splat << 16) | Splat;
1225 if (VT == MVT::i64)
1226 Imm = Splat << 32 | Splat;
1227 else
1228 Imm = SignExtend64<32>(x: Splat);
1229 } else {
1230 // If the upper 32-bits are not used try to convert this into a simm32 by
1231 // sign extending bit 32.
1232 if (!isInt<32>(x: Imm) && isUInt<32>(x: Imm) && hasAllWUsers(Node))
1233 Imm = SignExtend64<32>(x: Imm);
1234
1235 if (VT == MVT::i64 && !isInt<12>(x: Imm) && !isShiftedInt<20, 12>(x: Imm) &&
1236 Subtarget->hasStdExtP() && isApplicableToPLIOrPLUI(Val: Imm) &&
1237 hasAllWUsers(Node)) {
1238 // If it's 4 packed 8-bit integers or 2 packed signed 16-bit integers,
1239 // we can simply copy lower 32 bits to higher 32 bits to make it able to
1240 // rematerialize to PLI_B or PLI_H
1241 Imm = ((uint64_t)Imm << 32) | (Imm & 0xFFFFFFFF);
1242 }
1243 }
1244
1245 ReplaceNode(F: Node, T: selectImm(CurDAG, DL, VT, Imm, Subtarget: *Subtarget).getNode());
1246 return;
1247 }
1248 case ISD::ConstantFP: {
1249 const APFloat &APF = cast<ConstantFPSDNode>(Val: Node)->getValueAPF();
1250
1251 bool Is64Bit = Subtarget->is64Bit();
1252 bool HasZdinx = Subtarget->hasStdExtZdinx();
1253
1254 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
1255 SDValue Imm;
1256 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
1257 // create an integer immediate.
1258 if (APF.isPosZero() || NegZeroF64) {
1259 if (VT == MVT::f64 && HasZdinx && !Is64Bit)
1260 Imm = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::f64);
1261 else
1262 Imm = CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT);
1263 } else {
1264 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
1265 Subtarget: *Subtarget);
1266 }
1267
1268 unsigned Opc;
1269 switch (VT.SimpleTy) {
1270 default:
1271 llvm_unreachable("Unexpected size");
1272 case MVT::bf16:
1273 assert(Subtarget->hasStdExtZfbfmin());
1274 Opc = RISCV::FMV_H_X;
1275 break;
1276 case MVT::f16:
1277 Opc = Subtarget->hasStdExtZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
1278 break;
1279 case MVT::f32:
1280 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
1281 break;
1282 case MVT::f64:
1283 // For RV32, we can't move from a GPR, we need to convert instead. This
1284 // should only happen for +0.0 and -0.0.
1285 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
1286 if (HasZdinx)
1287 Opc = RISCV::COPY;
1288 else
1289 Opc = Is64Bit ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
1290 break;
1291 }
1292
1293 SDNode *Res;
1294 if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
1295 Res =
1296 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_16, DL, VT, Operand: Imm).getNode();
1297 } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
1298 Res =
1299 CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_32, DL, VT, Operand: Imm).getNode();
1300 } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
1301 Res = CurDAG->getMachineNode(
1302 Opcode: Opc, dl: DL, VT, Op1: Imm,
1303 Op2: CurDAG->getTargetConstant(Val: RISCVFPRndMode::RNE, DL, VT: XLenVT));
1304 else
1305 Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: Imm);
1306
1307 // For f64 -0.0, we need to insert a fneg.d idiom.
1308 if (NegZeroF64) {
1309 Opc = RISCV::FSGNJN_D;
1310 if (HasZdinx)
1311 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
1312 Res =
1313 CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: SDValue(Res, 0), Op2: SDValue(Res, 0));
1314 }
1315
1316 ReplaceNode(F: Node, T: Res);
1317 return;
1318 }
1319 case RISCVISD::BuildGPRPair:
1320 case RISCVISD::BuildPairF64:
1321 case RISCVISD::BuildPairGPRVec: {
1322 if (Opcode == RISCVISD::BuildPairF64 && !Subtarget->hasStdExtZdinx())
1323 break;
1324
1325 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::BuildPairF64) &&
1326 "BuildPairF64 only handled here on rv32i_zdinx");
1327
1328 SDValue N =
1329 buildGPRPair(CurDAG, DL, VT, Lo: Node->getOperand(Num: 0), Hi: Node->getOperand(Num: 1));
1330 ReplaceNode(F: Node, T: N.getNode());
1331 return;
1332 }
1333 case RISCVISD::SplitGPRPair:
1334 case RISCVISD::SplitF64:
1335 case RISCVISD::SplitGPRVec: {
1336 if (Subtarget->hasStdExtZdinx() || Opcode != RISCVISD::SplitF64) {
1337 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::SplitF64) &&
1338 "SplitF64 only handled here on rv32i_zdinx");
1339
1340 if (!SDValue(Node, 0).use_empty()) {
1341 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: RISCV::sub_gpr_even, DL,
1342 VT: Node->getValueType(ResNo: 0),
1343 Operand: Node->getOperand(Num: 0));
1344 ReplaceUses(F: SDValue(Node, 0), T: Lo);
1345 }
1346
1347 if (!SDValue(Node, 1).use_empty()) {
1348 SDValue Hi = CurDAG->getTargetExtractSubreg(
1349 SRIdx: RISCV::sub_gpr_odd, DL, VT: Node->getValueType(ResNo: 1), Operand: Node->getOperand(Num: 0));
1350 ReplaceUses(F: SDValue(Node, 1), T: Hi);
1351 }
1352
1353 CurDAG->RemoveDeadNode(N: Node);
1354 return;
1355 }
1356
1357 if (!Subtarget->hasStdExtZfa())
1358 break;
1359 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
1360 "Unexpected subtarget");
1361
1362 // With Zfa, lower to fmv.x.w and fmvh.x.d.
1363 if (!SDValue(Node, 0).use_empty()) {
1364 SDNode *Lo = CurDAG->getMachineNode(Opcode: RISCV::FMV_X_W_FPR64, dl: DL, VT,
1365 Op1: Node->getOperand(Num: 0));
1366 ReplaceUses(F: SDValue(Node, 0), T: SDValue(Lo, 0));
1367 }
1368 if (!SDValue(Node, 1).use_empty()) {
1369 SDNode *Hi = CurDAG->getMachineNode(Opcode: RISCV::FMVH_X_D, dl: DL, VT,
1370 Op1: Node->getOperand(Num: 0));
1371 ReplaceUses(F: SDValue(Node, 1), T: SDValue(Hi, 0));
1372 }
1373
1374 CurDAG->RemoveDeadNode(N: Node);
1375 return;
1376 }
1377 case ISD::SHL: {
1378 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1379 if (!N1C)
1380 break;
1381 SDValue N0 = Node->getOperand(Num: 0);
1382 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
1383 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1384 break;
1385 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1386 unsigned ShAmt = N1C->getZExtValue();
1387 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1388 "Unexpected shift amount!");
1389 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1390
1391 if (isShiftedMask_64(Value: Mask)) {
1392 unsigned XLen = Subtarget->getXLen();
1393 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1394 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1395 if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
1396 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
1397 // where C2 has 32 leading zeros and C3 trailing zeros.
1398 SDNode *SRLIW = CurDAG->getMachineNode(
1399 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1400 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1401 SDNode *SLLI = CurDAG->getMachineNode(
1402 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1403 Op2: CurDAG->getTargetConstant(Val: TrailingZeros + ShAmt, DL, VT));
1404 ReplaceNode(F: Node, T: SLLI);
1405 return;
1406 }
1407 if (TrailingZeros == 0 && LeadingZeros > ShAmt &&
1408 XLen - LeadingZeros > 11 && LeadingZeros != 32) {
1409 // Optimize (shl (and X, C2), C) -> (srli (slli X, C4), C4-C)
1410 // where C2 has C4 leading zeros and no trailing zeros.
1411 // This is profitable if the "and" was to be lowered to
1412 // (srli (slli X, C4), C4) and not (andi X, C2).
1413 // For "LeadingZeros == 32":
1414 // - with Zba it's just (slli.uw X, C)
1415 // - without Zba a tablegen pattern applies the very same
1416 // transform as we would have done here
1417 SDNode *SLLI = CurDAG->getMachineNode(
1418 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1419 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1420 SDNode *SRLI = CurDAG->getMachineNode(
1421 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1422 Op2: CurDAG->getTargetConstant(Val: LeadingZeros - ShAmt, DL, VT));
1423 ReplaceNode(F: Node, T: SRLI);
1424 return;
1425 }
1426 }
1427 break;
1428 }
1429 case ISD::SRL: {
1430 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1431 if (!N1C)
1432 break;
1433 SDValue N0 = Node->getOperand(Num: 0);
1434 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1435 break;
1436 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1437 unsigned ShAmt = N1C->getZExtValue();
1438 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1439 "Unexpected shift amount!");
1440 uint64_t Mask = N0.getConstantOperandVal(i: 1);
1441
1442 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1443 // 32 leading zeros and C3 trailing zeros.
1444 if (isShiftedMask_64(Value: Mask) && N0.hasOneUse()) {
1445 unsigned XLen = Subtarget->getXLen();
1446 unsigned LeadingZeros = XLen - llvm::bit_width(Value: Mask);
1447 unsigned TrailingZeros = llvm::countr_zero(Val: Mask);
1448 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1449 SDNode *SRLIW = CurDAG->getMachineNode(
1450 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
1451 Op2: CurDAG->getTargetConstant(Val: TrailingZeros, DL, VT));
1452 SDNode *SLLI = CurDAG->getMachineNode(
1453 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1454 Op2: CurDAG->getTargetConstant(Val: TrailingZeros - ShAmt, DL, VT));
1455 ReplaceNode(F: Node, T: SLLI);
1456 return;
1457 }
1458 }
1459
1460 // Optimize (srl (and X, C2), C) ->
1461 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1462 // Where C2 is a mask with C3 trailing ones.
1463 // Taking into account that the C2 may have had lower bits unset by
1464 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1465 // This pattern occurs when type legalizing right shifts for types with
1466 // less than XLen bits.
1467 Mask |= maskTrailingOnes<uint64_t>(N: ShAmt);
1468 if (!isMask_64(Value: Mask))
1469 break;
1470 unsigned TrailingOnes = llvm::countr_one(Value: Mask);
1471 if (ShAmt >= TrailingOnes)
1472 break;
1473 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1474 if (TrailingOnes == 32) {
1475 SDNode *SRLI = CurDAG->getMachineNode(
1476 Opcode: Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, dl: DL, VT,
1477 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1478 ReplaceNode(F: Node, T: SRLI);
1479 return;
1480 }
1481
1482 // Only do the remaining transforms if the AND has one use.
1483 if (!N0.hasOneUse())
1484 break;
1485
1486 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1487 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1488 SDNode *BEXTI = CurDAG->getMachineNode(
1489 Opcode: Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, dl: DL, VT,
1490 Op1: N0.getOperand(i: 0), Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT));
1491 ReplaceNode(F: Node, T: BEXTI);
1492 return;
1493 }
1494
1495 const unsigned Msb = TrailingOnes - 1;
1496 const unsigned Lsb = ShAmt;
1497 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0.getOperand(i: 0), Msb, Lsb))
1498 return;
1499
1500 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1501 SDNode *SLLI =
1502 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1503 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1504 SDNode *SRLI = CurDAG->getMachineNode(
1505 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1506 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1507 ReplaceNode(F: Node, T: SRLI);
1508 return;
1509 }
1510 case ISD::SRA: {
1511 if (trySignedBitfieldExtract(Node))
1512 return;
1513
1514 if (trySignedBitfieldInsertInSign(Node))
1515 return;
1516
1517 // Optimize (sra (sext_inreg X, i16), C) ->
1518 // (srai (slli X, (XLen-16), (XLen-16) + C)
1519 // And (sra (sext_inreg X, i8), C) ->
1520 // (srai (slli X, (XLen-8), (XLen-8) + C)
1521 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1522 // This transform matches the code we get without Zbb. The shifts are more
1523 // compressible, and this can help expose CSE opportunities in the sdiv by
1524 // constant optimization.
1525 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1526 if (!N1C)
1527 break;
1528 SDValue N0 = Node->getOperand(Num: 0);
1529 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1530 break;
1531 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1532 unsigned ShAmt = N1C->getZExtValue();
1533 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1534 "Unexpected shift amount!");
1535 unsigned ExtSize =
1536 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
1537 // ExtSize of 32 should use sraiw via tablegen pattern.
1538 if (ExtSize >= 32 || ShAmt >= ExtSize)
1539 break;
1540 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1541 SDNode *SLLI =
1542 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1543 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1544 SDNode *SRAI = CurDAG->getMachineNode(
1545 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1546 Op2: CurDAG->getTargetConstant(Val: LShAmt + ShAmt, DL, VT));
1547 ReplaceNode(F: Node, T: SRAI);
1548 return;
1549 }
1550 case ISD::SIGN_EXTEND_INREG: {
1551 // Optimize (sext_inreg (srl X, C), i8/i16) ->
1552 // (srai (slli X, XLen-ExtSize-C), XLen-ExtSize)
1553 // This is a bitfield extract pattern where we're extracting a signed
1554 // 8-bit or 16-bit field from position C.
1555 SDValue N0 = Node->getOperand(Num: 0);
1556 if (N0.getOpcode() != ISD::SRL || !N0.hasOneUse())
1557 break;
1558
1559 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1560 if (!ShAmtC)
1561 break;
1562
1563 unsigned ExtSize =
1564 cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT().getSizeInBits();
1565 unsigned ShAmt = ShAmtC->getZExtValue();
1566 unsigned XLen = Subtarget->getXLen();
1567
1568 // Only handle types less than 32, and make sure the shift amount is valid.
1569 if (ExtSize >= 32 || ShAmt >= XLen - ExtSize)
1570 break;
1571
1572 unsigned LShAmt = XLen - ExtSize - ShAmt;
1573 SDNode *SLLI =
1574 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1575 Op2: CurDAG->getTargetConstant(Val: LShAmt, DL, VT));
1576 SDNode *SRAI = CurDAG->getMachineNode(
1577 Opcode: RISCV::SRAI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1578 Op2: CurDAG->getTargetConstant(Val: XLen - ExtSize, DL, VT));
1579 ReplaceNode(F: Node, T: SRAI);
1580 return;
1581 }
1582 case ISD::OR: {
1583 if (tryShrinkShlLogicImm(Node))
1584 return;
1585
1586 break;
1587 }
1588 case ISD::XOR:
1589 if (tryShrinkShlLogicImm(Node))
1590 return;
1591
1592 break;
1593 case ISD::AND: {
1594 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1595 if (!N1C)
1596 break;
1597 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1598
1599 SDValue N0 = Node->getOperand(Num: 0);
1600
1601 bool LeftShift = N0.getOpcode() == ISD::SHL;
1602 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1603 auto *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
1604 if (!C)
1605 break;
1606 unsigned C2 = C->getZExtValue();
1607 unsigned XLen = Subtarget->getXLen();
1608 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1609
1610 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1611 // shift pair might offer more compression opportunities.
1612 // TODO: We could check for C extension here, but we don't have many lit
1613 // tests with the C extension enabled so not checking gets better
1614 // coverage.
1615 // TODO: What if ANDI faster than shift?
1616 bool IsCANDI = isInt<6>(x: N1C->getSExtValue());
1617
1618 uint64_t C1 = N1C->getZExtValue();
1619
1620 // Clear irrelevant bits in the mask.
1621 if (LeftShift)
1622 C1 &= maskTrailingZeros<uint64_t>(N: C2);
1623 else
1624 C1 &= maskTrailingOnes<uint64_t>(N: XLen - C2);
1625
1626 // Some transforms should only be done if the shift has a single use or
1627 // the AND would become (srli (slli X, 32), 32)
1628 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1629
1630 SDValue X = N0.getOperand(i: 0);
1631
1632 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1633 // with c3 leading zeros.
1634 if (!LeftShift && isMask_64(Value: C1)) {
1635 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1636 if (C2 < Leading) {
1637 // If the number of leading zeros is C2+32 this can be SRLIW.
1638 if (C2 + 32 == Leading) {
1639 SDNode *SRLIW = CurDAG->getMachineNode(
1640 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X, Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1641 ReplaceNode(F: Node, T: SRLIW);
1642 return;
1643 }
1644
1645 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1646 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1647 //
1648 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1649 // legalized and goes through DAG combine.
1650 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1651 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1652 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32) {
1653 SDNode *SRAIW =
1654 CurDAG->getMachineNode(Opcode: RISCV::SRAIW, dl: DL, VT, Op1: X.getOperand(i: 0),
1655 Op2: CurDAG->getTargetConstant(Val: 31, DL, VT));
1656 SDNode *SRLIW = CurDAG->getMachineNode(
1657 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: SDValue(SRAIW, 0),
1658 Op2: CurDAG->getTargetConstant(Val: Leading - 32, DL, VT));
1659 ReplaceNode(F: Node, T: SRLIW);
1660 return;
1661 }
1662
1663 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1664 // available.
1665 // Transform (and (srl x, C2), C1)
1666 // -> (<bfextract> x, msb, lsb)
1667 //
1668 // Make sure to keep this below the SRLIW cases, as we always want to
1669 // prefer the more common instruction.
1670 const unsigned Msb = llvm::bit_width(Value: C1) + C2 - 1;
1671 const unsigned Lsb = C2;
1672 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1673 return;
1674
1675 // (srli (slli x, c3-c2), c3).
1676 // Skip if we could use (zext.w (sraiw X, C2)).
1677 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1678 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1679 cast<VTSDNode>(Val: X.getOperand(i: 1))->getVT() == MVT::i32;
1680 // Also Skip if we can use bexti or th.tst.
1681 Skip |= HasBitTest && Leading == XLen - 1;
1682 if (OneUseOrZExtW && !Skip) {
1683 SDNode *SLLI = CurDAG->getMachineNode(
1684 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1685 Op2: CurDAG->getTargetConstant(Val: Leading - C2, DL, VT));
1686 SDNode *SRLI = CurDAG->getMachineNode(
1687 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1688 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1689 ReplaceNode(F: Node, T: SRLI);
1690 return;
1691 }
1692 }
1693 }
1694
1695 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1696 // shifted by c2 bits with c3 leading zeros.
1697 if (LeftShift && isShiftedMask_64(Value: C1)) {
1698 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1699
1700 if (C2 + Leading < XLen &&
1701 C1 == (maskTrailingOnes<uint64_t>(N: XLen - (C2 + Leading)) << C2)) {
1702 // Use slli.uw when possible.
1703 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1704 SDNode *SLLI_UW =
1705 CurDAG->getMachineNode(Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: X,
1706 Op2: CurDAG->getTargetConstant(Val: C2, DL, VT));
1707 ReplaceNode(F: Node, T: SLLI_UW);
1708 return;
1709 }
1710
1711 // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
1712 // available.
1713 // Transform (and (shl x, c2), c1)
1714 // -> (<bfinsert> x, msb, lsb)
1715 // e.g.
1716 // (and (shl x, 12), 0x00fff000)
1717 // If XLen = 32 and C2 = 12, then
1718 // Msb = 32 - 8 - 1 = 23 and Lsb = 12
1719 const unsigned Msb = XLen - Leading - 1;
1720 const unsigned Lsb = C2;
1721 if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
1722 return;
1723
1724 if (OneUseOrZExtW && !IsCANDI) {
1725 // (packh x0, X)
1726 if (Subtarget->hasStdExtZbkb() && C1 == 0xff00 && C2 == 8) {
1727 SDNode *PACKH = CurDAG->getMachineNode(
1728 Opcode: RISCV::PACKH, dl: DL, VT,
1729 Op1: CurDAG->getRegister(Reg: RISCV::X0, VT: Subtarget->getXLenVT()), Op2: X);
1730 ReplaceNode(F: Node, T: PACKH);
1731 return;
1732 }
1733 // (srli (slli c2+c3), c3)
1734 SDNode *SLLI = CurDAG->getMachineNode(
1735 Opcode: RISCV::SLLI, dl: DL, VT, Op1: X,
1736 Op2: CurDAG->getTargetConstant(Val: C2 + Leading, DL, VT));
1737 SDNode *SRLI = CurDAG->getMachineNode(
1738 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SLLI, 0),
1739 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1740 ReplaceNode(F: Node, T: SRLI);
1741 return;
1742 }
1743 }
1744 }
1745
1746 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1747 // shifted mask with c2 leading zeros and c3 trailing zeros.
1748 if (!LeftShift && isShiftedMask_64(Value: C1)) {
1749 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1750 unsigned Trailing = llvm::countr_zero(Val: C1);
1751 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1752 !IsCANDI) {
1753 unsigned SrliOpc = RISCV::SRLI;
1754 // If the input is zexti32 we should use SRLIW.
1755 if (X.getOpcode() == ISD::AND &&
1756 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1757 X.getConstantOperandVal(i: 1) == UINT64_C(0xFFFFFFFF)) {
1758 SrliOpc = RISCV::SRLIW;
1759 X = X.getOperand(i: 0);
1760 }
1761 SDNode *SRLI = CurDAG->getMachineNode(
1762 Opcode: SrliOpc, dl: DL, VT, Op1: X,
1763 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1764 SDNode *SLLI = CurDAG->getMachineNode(
1765 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1766 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1767 ReplaceNode(F: Node, T: SLLI);
1768 return;
1769 }
1770 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1771 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1772 OneUseOrZExtW && !IsCANDI) {
1773 SDNode *SRLIW = CurDAG->getMachineNode(
1774 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1775 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1776 SDNode *SLLI = CurDAG->getMachineNode(
1777 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1778 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1779 ReplaceNode(F: Node, T: SLLI);
1780 return;
1781 }
1782 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1783 if (Trailing > 0 && Leading + Trailing == 32 && C2 + Trailing < XLen &&
1784 OneUseOrZExtW && Subtarget->hasStdExtZba()) {
1785 SDNode *SRLI = CurDAG->getMachineNode(
1786 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1787 Op2: CurDAG->getTargetConstant(Val: C2 + Trailing, DL, VT));
1788 SDNode *SLLI_UW = CurDAG->getMachineNode(
1789 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1790 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1791 ReplaceNode(F: Node, T: SLLI_UW);
1792 return;
1793 }
1794 }
1795
1796 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1797 // shifted mask with no leading zeros and c3 trailing zeros.
1798 if (LeftShift && isShiftedMask_64(Value: C1)) {
1799 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1800 unsigned Trailing = llvm::countr_zero(Val: C1);
1801 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1802 SDNode *SRLI = CurDAG->getMachineNode(
1803 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1804 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1805 SDNode *SLLI = CurDAG->getMachineNode(
1806 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1807 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1808 ReplaceNode(F: Node, T: SLLI);
1809 return;
1810 }
1811 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1812 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1813 SDNode *SRLIW = CurDAG->getMachineNode(
1814 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: X,
1815 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1816 SDNode *SLLI = CurDAG->getMachineNode(
1817 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLIW, 0),
1818 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1819 ReplaceNode(F: Node, T: SLLI);
1820 return;
1821 }
1822
1823 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1824 if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
1825 Subtarget->hasStdExtZba()) {
1826 SDNode *SRLI = CurDAG->getMachineNode(
1827 Opcode: RISCV::SRLI, dl: DL, VT, Op1: X,
1828 Op2: CurDAG->getTargetConstant(Val: Trailing - C2, DL, VT));
1829 SDNode *SLLI_UW = CurDAG->getMachineNode(
1830 Opcode: RISCV::SLLI_UW, dl: DL, VT, Op1: SDValue(SRLI, 0),
1831 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1832 ReplaceNode(F: Node, T: SLLI_UW);
1833 return;
1834 }
1835 }
1836 }
1837
1838 const uint64_t C1 = N1C->getZExtValue();
1839
1840 if (N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
1841 N0.hasOneUse()) {
1842 unsigned C2 = N0.getConstantOperandVal(i: 1);
1843 unsigned XLen = Subtarget->getXLen();
1844 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1845
1846 SDValue X = N0.getOperand(i: 0);
1847
1848 // Prefer SRAIW + ANDI when possible.
1849 bool Skip = C2 > 32 && isInt<12>(x: N1C->getSExtValue()) &&
1850 X.getOpcode() == ISD::SHL &&
1851 isa<ConstantSDNode>(Val: X.getOperand(i: 1)) &&
1852 X.getConstantOperandVal(i: 1) == 32;
1853 // Turn (and (sra x, c2), c1) -> (srli (srai x, c2-c3), c3) if c1 is a
1854 // mask with c3 leading zeros and c2 is larger than c3.
1855 if (isMask_64(Value: C1) && !Skip) {
1856 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1857 if (C2 > Leading) {
1858 SDNode *SRAI = CurDAG->getMachineNode(
1859 Opcode: RISCV::SRAI, dl: DL, VT, Op1: X,
1860 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1861 SDNode *SRLI = CurDAG->getMachineNode(
1862 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1863 Op2: CurDAG->getTargetConstant(Val: Leading, DL, VT));
1864 ReplaceNode(F: Node, T: SRLI);
1865 return;
1866 }
1867 }
1868
1869 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
1870 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
1871 // use (slli (srli (srai y, c2 - c3), c3 + c4), c4).
1872 if (isShiftedMask_64(Value: C1) && !Skip) {
1873 unsigned Leading = XLen - llvm::bit_width(Value: C1);
1874 unsigned Trailing = llvm::countr_zero(Val: C1);
1875 if (C2 > Leading && Leading > 0 && Trailing > 0) {
1876 SDNode *SRAI = CurDAG->getMachineNode(
1877 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1878 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT));
1879 SDNode *SRLI = CurDAG->getMachineNode(
1880 Opcode: RISCV::SRLI, dl: DL, VT, Op1: SDValue(SRAI, 0),
1881 Op2: CurDAG->getTargetConstant(Val: Leading + Trailing, DL, VT));
1882 SDNode *SLLI = CurDAG->getMachineNode(
1883 Opcode: RISCV::SLLI, dl: DL, VT, Op1: SDValue(SRLI, 0),
1884 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT));
1885 ReplaceNode(F: Node, T: SLLI);
1886 return;
1887 }
1888 }
1889 }
1890
1891 // If C1 masks off the upper bits only (but can't be formed as an
1892 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1893 // available.
1894 // Transform (and x, C1)
1895 // -> (<bfextract> x, msb, lsb)
1896 if (isMask_64(Value: C1) && !isInt<12>(x: N1C->getSExtValue()) &&
1897 !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
1898 !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
1899 const unsigned Msb = llvm::bit_width(Value: C1) - 1;
1900 if (tryUnsignedBitfieldExtract(Node, DL, VT, X: N0, Msb, Lsb: 0))
1901 return;
1902 }
1903
1904 if (tryShrinkShlLogicImm(Node))
1905 return;
1906
1907 break;
1908 }
1909 case ISD::MUL: {
1910 // Special case for calculating (mul (and X, C2), C1) where the full product
1911 // fits in XLen bits. We can shift X left by the number of leading zeros in
1912 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1913 // product has XLen trailing zeros, putting it in the output of MULHU. This
1914 // can avoid materializing a constant in a register for C2.
1915
1916 // RHS should be a constant.
1917 auto *N1C = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 1));
1918 if (!N1C || !N1C->hasOneUse())
1919 break;
1920
1921 // LHS should be an AND with constant.
1922 SDValue N0 = Node->getOperand(Num: 0);
1923 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
1924 break;
1925
1926 uint64_t C2 = N0.getConstantOperandVal(i: 1);
1927
1928 // Constant should be a mask.
1929 if (!isMask_64(Value: C2))
1930 break;
1931
1932 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1933 // multiple users or the constant is a simm12. This prevents inserting a
1934 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1935 // make it more costly to materialize. Otherwise, using a SLLI might allow
1936 // it to be compressed.
1937 bool IsANDIOrZExt =
1938 isInt<12>(x: C2) ||
1939 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1940 // With XTHeadBb, we can use TH.EXTU.
1941 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1942 if (IsANDIOrZExt && (isInt<12>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1943 break;
1944 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1945 // the constant is a simm32.
1946 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1947 // With XTHeadBb, we can use TH.EXTU.
1948 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1949 if (IsZExtW && (isInt<32>(x: N1C->getSExtValue()) || !N0.hasOneUse()))
1950 break;
1951
1952 // We need to shift left the AND input and C1 by a total of XLen bits.
1953
1954 // How far left do we need to shift the AND input?
1955 unsigned XLen = Subtarget->getXLen();
1956 unsigned LeadingZeros = XLen - llvm::bit_width(Value: C2);
1957
1958 // The constant gets shifted by the remaining amount unless that would
1959 // shift bits out.
1960 uint64_t C1 = N1C->getZExtValue();
1961 unsigned ConstantShift = XLen - LeadingZeros;
1962 if (ConstantShift > (XLen - llvm::bit_width(Value: C1)))
1963 break;
1964
1965 uint64_t ShiftedC1 = C1 << ConstantShift;
1966 // If this RV32, we need to sign extend the constant.
1967 if (XLen == 32)
1968 ShiftedC1 = SignExtend64<32>(x: ShiftedC1);
1969
1970 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1971 SDNode *Imm = selectImm(CurDAG, DL, VT, Imm: ShiftedC1, Subtarget: *Subtarget).getNode();
1972 SDNode *SLLI =
1973 CurDAG->getMachineNode(Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
1974 Op2: CurDAG->getTargetConstant(Val: LeadingZeros, DL, VT));
1975 SDNode *MULHU = CurDAG->getMachineNode(Opcode: RISCV::MULHU, dl: DL, VT,
1976 Op1: SDValue(SLLI, 0), Op2: SDValue(Imm, 0));
1977 ReplaceNode(F: Node, T: MULHU);
1978 return;
1979 }
1980 case ISD::SMUL_LOHI:
1981 case ISD::UMUL_LOHI:
1982 case RISCVISD::WMULSU:
1983 case RISCVISD::WADD:
1984 case RISCVISD::WSUB:
1985 case RISCVISD::WADDU:
1986 case RISCVISD::WSUBU: {
1987 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1988 "Unexpected opcode");
1989
1990 unsigned Opc;
1991 switch (Node->getOpcode()) {
1992 default:
1993 llvm_unreachable("Unexpected opcode");
1994 case ISD::SMUL_LOHI:
1995 Opc = RISCV::WMUL;
1996 break;
1997 case ISD::UMUL_LOHI:
1998 Opc = RISCV::WMULU;
1999 break;
2000 case RISCVISD::WMULSU:
2001 Opc = RISCV::WMULSU;
2002 break;
2003 case RISCVISD::WADD:
2004 Opc = RISCV::WADD;
2005 break;
2006 case RISCVISD::WSUB:
2007 Opc = RISCV::WSUB;
2008 break;
2009 case RISCVISD::WADDU:
2010 Opc = RISCV::WADDU;
2011 break;
2012 case RISCVISD::WSUBU:
2013 Opc = RISCV::WSUBU;
2014 break;
2015 }
2016
2017 SDNode *Result = CurDAG->getMachineNode(
2018 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Node->getOperand(Num: 0), Op2: Node->getOperand(Num: 1));
2019
2020 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(Result, 0));
2021 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2022 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2023 CurDAG->RemoveDeadNode(N: Node);
2024 return;
2025 }
2026 case RISCVISD::WSLL:
2027 case RISCVISD::WSLA: {
2028 // Custom select WSLL/WSLA for RV32P.
2029 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
2030 "Unexpected opcode");
2031
2032 bool IsSigned = Node->getOpcode() == RISCVISD::WSLA;
2033
2034 SDValue ShAmt = Node->getOperand(Num: 1);
2035
2036 unsigned Opc;
2037
2038 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShAmt);
2039 if (ShAmtC && ShAmtC->getZExtValue() < 64) {
2040 Opc = IsSigned ? RISCV::WSLAI : RISCV::WSLLI;
2041 ShAmt = CurDAG->getTargetConstant(Val: ShAmtC->getZExtValue(), DL, VT: XLenVT);
2042 } else {
2043 Opc = IsSigned ? RISCV::WSLA : RISCV::WSLL;
2044 }
2045
2046 SDNode *WShift = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped,
2047 Op1: Node->getOperand(Num: 0), Op2: ShAmt);
2048
2049 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(WShift, 0));
2050 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2051 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2052 CurDAG->RemoveDeadNode(N: Node);
2053 return;
2054 }
2055 case ISD::LOAD: {
2056 if (tryIndexedLoad(Node))
2057 return;
2058
2059 if (Subtarget->hasVendorXCVmem() && !Subtarget->is64Bit()) {
2060 // We match post-incrementing load here
2061 LoadSDNode *Load = cast<LoadSDNode>(Val: Node);
2062 if (Load->getAddressingMode() != ISD::POST_INC)
2063 break;
2064
2065 SDValue Chain = Node->getOperand(Num: 0);
2066 SDValue Base = Node->getOperand(Num: 1);
2067 SDValue Offset = Node->getOperand(Num: 2);
2068
2069 bool Simm12 = false;
2070 bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
2071
2072 if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Val&: Offset)) {
2073 int ConstantVal = ConstantOffset->getSExtValue();
2074 Simm12 = isInt<12>(x: ConstantVal);
2075 if (Simm12)
2076 Offset = CurDAG->getSignedTargetConstant(Val: ConstantVal, DL: SDLoc(Offset),
2077 VT: Offset.getValueType());
2078 }
2079
2080 unsigned Opcode = 0;
2081 switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
2082 case MVT::i8:
2083 if (Simm12 && SignExtend)
2084 Opcode = RISCV::CV_LB_ri_inc;
2085 else if (Simm12 && !SignExtend)
2086 Opcode = RISCV::CV_LBU_ri_inc;
2087 else if (!Simm12 && SignExtend)
2088 Opcode = RISCV::CV_LB_rr_inc;
2089 else
2090 Opcode = RISCV::CV_LBU_rr_inc;
2091 break;
2092 case MVT::i16:
2093 if (Simm12 && SignExtend)
2094 Opcode = RISCV::CV_LH_ri_inc;
2095 else if (Simm12 && !SignExtend)
2096 Opcode = RISCV::CV_LHU_ri_inc;
2097 else if (!Simm12 && SignExtend)
2098 Opcode = RISCV::CV_LH_rr_inc;
2099 else
2100 Opcode = RISCV::CV_LHU_rr_inc;
2101 break;
2102 case MVT::i32:
2103 if (Simm12)
2104 Opcode = RISCV::CV_LW_ri_inc;
2105 else
2106 Opcode = RISCV::CV_LW_rr_inc;
2107 break;
2108 default:
2109 break;
2110 }
2111 if (!Opcode)
2112 break;
2113
2114 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode, dl: DL, VT1: XLenVT, VT2: XLenVT,
2115 VT3: Chain.getSimpleValueType(), Op1: Base,
2116 Op2: Offset, Op3: Chain));
2117 return;
2118 }
2119 break;
2120 }
2121 case RISCVISD::LD_RV32: {
2122 assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd");
2123
2124 SDValue Base, Offset;
2125 SDValue Chain = Node->getOperand(Num: 0);
2126 SDValue Addr = Node->getOperand(Num: 1);
2127 SelectAddrRegImm(Addr, Base, Offset);
2128
2129 SDValue Ops[] = {Base, Offset, Chain};
2130 MachineSDNode *New = CurDAG->getMachineNode(
2131 Opcode: RISCV::LD_RV32, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2132 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2133 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2134 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2135 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2136 ReplaceUses(F: SDValue(Node, 2), T: SDValue(New, 1));
2137 CurDAG->RemoveDeadNode(N: Node);
2138 return;
2139 }
2140 case RISCVISD::SD_RV32: {
2141 SDValue Base, Offset;
2142 SDValue Chain = Node->getOperand(Num: 0);
2143 SDValue Addr = Node->getOperand(Num: 3);
2144 SelectAddrRegImm(Addr, Base, Offset);
2145
2146 SDValue Lo = Node->getOperand(Num: 1);
2147 SDValue Hi = Node->getOperand(Num: 2);
2148
2149 SDValue RegPair;
2150 // Peephole to use X0_Pair for storing zero.
2151 if (isNullConstant(V: Lo) && isNullConstant(V: Hi)) {
2152 RegPair = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2153 } else {
2154 RegPair = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo, Hi);
2155 }
2156
2157 MachineSDNode *New = CurDAG->getMachineNode(Opcode: RISCV::SD_RV32, dl: DL, VT: MVT::Other,
2158 Ops: {RegPair, Base, Offset, Chain});
2159 CurDAG->setNodeMemRefs(N: New, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2160 ReplaceUses(F: SDValue(Node, 0), T: SDValue(New, 0));
2161 CurDAG->RemoveDeadNode(N: Node);
2162 return;
2163 }
2164 case RISCVISD::MQWACC:
2165 case RISCVISD::MQRWACC:
2166 case RISCVISD::WMACC:
2167 case RISCVISD::WMACCU:
2168 case RISCVISD::WMACCSU: {
2169 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2170 "Unexpected opcode");
2171
2172 SDValue Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Node->getOperand(Num: 0),
2173 Hi: Node->getOperand(Num: 1));
2174 unsigned Opc;
2175 switch (Opcode) {
2176 default:
2177 llvm_unreachable("Unexpected opcode");
2178 case RISCVISD::MQWACC:
2179 Opc = RISCV::MQWACC;
2180 break;
2181 case RISCVISD::MQRWACC:
2182 Opc = RISCV::MQRWACC;
2183 break;
2184 case RISCVISD::WMACC:
2185 Opc = RISCV::WMACC;
2186 break;
2187 case RISCVISD::WMACCU:
2188 Opc = RISCV::WMACCU;
2189 break;
2190 case RISCVISD::WMACCSU:
2191 Opc = RISCV::WMACCSU;
2192 break;
2193 }
2194 MachineSDNode *New = CurDAG->getMachineNode(
2195 Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Node->getOperand(Num: 2), Op3: Node->getOperand(Num: 3));
2196 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2197 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2198 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2199 CurDAG->RemoveDeadNode(N: Node);
2200 return;
2201 }
2202 case RISCVISD::ADDD:
2203 // Try to match WMACC pattern: ADDD where one operand pair comes from a
2204 // widening multiply.
2205 if (tryWideningMulAcc(Node, DL))
2206 return;
2207
2208 // Fall through to regular ADDD selection.
2209 [[fallthrough]];
2210 case RISCVISD::SUBD:
2211 case RISCVISD::WADDAU:
2212 case RISCVISD::WSUBAU:
2213 case RISCVISD::WADDA:
2214 case RISCVISD::WSUBA: {
2215 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2216 "Unexpected opcode");
2217
2218 SDValue Op0Lo = Node->getOperand(Num: 0);
2219 SDValue Op0Hi = Node->getOperand(Num: 1);
2220
2221 SDValue Op0;
2222 if (isNullConstant(V: Op0Lo) && isNullConstant(V: Op0Hi)) {
2223 Op0 = CurDAG->getRegister(Reg: RISCV::X0_Pair, VT: MVT::Untyped);
2224 } else {
2225 Op0 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op0Lo, Hi: Op0Hi);
2226 }
2227
2228 SDValue Op1Lo = Node->getOperand(Num: 2);
2229 SDValue Op1Hi = Node->getOperand(Num: 3);
2230
2231 MachineSDNode *New;
2232 if (Opcode == RISCVISD::WADDAU || Opcode == RISCVISD::WSUBAU ||
2233 Opcode == RISCVISD::WADDA || Opcode == RISCVISD::WSUBA) {
2234 // Widening accumulate: Op0 is the accumulator (GPRPair), Op1Lo and Op1Hi
2235 // are the two 32-bit values.
2236 unsigned Opc;
2237 switch (Opcode) {
2238 default:
2239 llvm_unreachable("Unexpected opcode");
2240 case RISCVISD::WADDAU:
2241 Opc = RISCV::WADDAU;
2242 break;
2243 case RISCVISD::WSUBAU:
2244 Opc = RISCV::WSUBAU;
2245 break;
2246 case RISCVISD::WADDA:
2247 Opc = RISCV::WADDA;
2248 break;
2249 case RISCVISD::WSUBA:
2250 Opc = RISCV::WSUBA;
2251 break;
2252 }
2253 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1Lo, Op3: Op1Hi);
2254 } else {
2255 SDValue Op1 = buildGPRPair(CurDAG, DL, VT: MVT::Untyped, Lo: Op1Lo, Hi: Op1Hi);
2256
2257 unsigned Opc;
2258 switch (Opcode) {
2259 default:
2260 llvm_unreachable("Unexpected opcode");
2261 case RISCVISD::ADDD:
2262 Opc = RISCV::ADDD;
2263 break;
2264 case RISCVISD::SUBD:
2265 Opc = RISCV::SUBD;
2266 break;
2267 }
2268 New = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Op1: Op0, Op2: Op1);
2269 }
2270
2271 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, Pair: SDValue(New, 0));
2272 ReplaceUses(F: SDValue(Node, 0), T: Lo);
2273 ReplaceUses(F: SDValue(Node, 1), T: Hi);
2274 CurDAG->RemoveDeadNode(N: Node);
2275 return;
2276 }
2277 case ISD::INTRINSIC_WO_CHAIN: {
2278 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
2279 switch (IntNo) {
2280 // By default we do not custom select any intrinsic.
2281 default:
2282 break;
2283 case Intrinsic::riscv_vmsgeu:
2284 case Intrinsic::riscv_vmsge: {
2285 SDValue Src1 = Node->getOperand(Num: 1);
2286 SDValue Src2 = Node->getOperand(Num: 2);
2287 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
2288 bool IsCmpConstant = false;
2289 bool IsCmpMinimum = false;
2290 // Only custom select scalar second operand.
2291 if (Src2.getValueType() != XLenVT)
2292 break;
2293 // Small constants are handled with patterns.
2294 int64_t CVal = 0;
2295 MVT Src1VT = Src1.getSimpleValueType();
2296 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2297 IsCmpConstant = true;
2298 CVal = C->getSExtValue();
2299 if (CVal >= -15 && CVal <= 16) {
2300 if (!IsUnsigned || CVal != 0)
2301 break;
2302 IsCmpMinimum = true;
2303 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2304 numBits: Src1VT.getScalarSizeInBits())
2305 .getSExtValue()) {
2306 IsCmpMinimum = true;
2307 }
2308 }
2309 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode, VMSGTOpcode;
2310 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2311 default:
2312 llvm_unreachable("Unexpected LMUL!");
2313#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2314 case RISCVVType::lmulenum: \
2315 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2316 : RISCV::PseudoVMSLT_VX_##suffix; \
2317 VMSGTOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix \
2318 : RISCV::PseudoVMSGT_VX_##suffix; \
2319 break;
2320 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2321 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2322 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2323 CASE_VMSLT_OPCODES(LMUL_1, M1)
2324 CASE_VMSLT_OPCODES(LMUL_2, M2)
2325 CASE_VMSLT_OPCODES(LMUL_4, M4)
2326 CASE_VMSLT_OPCODES(LMUL_8, M8)
2327#undef CASE_VMSLT_OPCODES
2328 }
2329 // Mask operations use the LMUL from the mask type.
2330 switch (RISCVTargetLowering::getLMUL(VT)) {
2331 default:
2332 llvm_unreachable("Unexpected LMUL!");
2333#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix) \
2334 case RISCVVType::lmulenum: \
2335 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
2336 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix; \
2337 break;
2338 CASE_VMNAND_VMSET_OPCODES(LMUL_F8, B64)
2339 CASE_VMNAND_VMSET_OPCODES(LMUL_F4, B32)
2340 CASE_VMNAND_VMSET_OPCODES(LMUL_F2, B16)
2341 CASE_VMNAND_VMSET_OPCODES(LMUL_1, B8)
2342 CASE_VMNAND_VMSET_OPCODES(LMUL_2, B4)
2343 CASE_VMNAND_VMSET_OPCODES(LMUL_4, B2)
2344 CASE_VMNAND_VMSET_OPCODES(LMUL_8, B1)
2345#undef CASE_VMNAND_VMSET_OPCODES
2346 }
2347 SDValue SEW = CurDAG->getTargetConstant(
2348 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2349 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2350 SDValue VL;
2351 selectVLOp(N: Node->getOperand(Num: 3), VL);
2352
2353 // If vmsge(u) with minimum value, expand it to vmset.
2354 if (IsCmpMinimum) {
2355 ReplaceNode(F: Node,
2356 T: CurDAG->getMachineNode(Opcode: VMSetOpcode, dl: DL, VT, Op1: VL, Op2: MaskSEW));
2357 return;
2358 }
2359
2360 if (IsCmpConstant) {
2361 SDValue Imm =
2362 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2363
2364 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMSGTOpcode, dl: DL, VT,
2365 Ops: {Src1, Imm, VL, SEW}));
2366 return;
2367 }
2368
2369 // Expand to
2370 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
2371 SDValue Cmp = SDValue(
2372 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2373 0);
2374 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMNANDOpcode, dl: DL, VT,
2375 Ops: {Cmp, Cmp, VL, MaskSEW}));
2376 return;
2377 }
2378 case Intrinsic::riscv_vmsgeu_mask:
2379 case Intrinsic::riscv_vmsge_mask: {
2380 SDValue Src1 = Node->getOperand(Num: 2);
2381 SDValue Src2 = Node->getOperand(Num: 3);
2382 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
2383 bool IsCmpConstant = false;
2384 bool IsCmpMinimum = false;
2385 // Only custom select scalar second operand.
2386 if (Src2.getValueType() != XLenVT)
2387 break;
2388 // Small constants are handled with patterns.
2389 MVT Src1VT = Src1.getSimpleValueType();
2390 int64_t CVal = 0;
2391 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Src2)) {
2392 IsCmpConstant = true;
2393 CVal = C->getSExtValue();
2394 if (CVal >= -15 && CVal <= 16) {
2395 if (!IsUnsigned || CVal != 0)
2396 break;
2397 IsCmpMinimum = true;
2398 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2399 numBits: Src1VT.getScalarSizeInBits())
2400 .getSExtValue()) {
2401 IsCmpMinimum = true;
2402 }
2403 }
2404 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
2405 VMOROpcode, VMSGTMaskOpcode;
2406 switch (RISCVTargetLowering::getLMUL(VT: Src1VT)) {
2407 default:
2408 llvm_unreachable("Unexpected LMUL!");
2409#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2410 case RISCVVType::lmulenum: \
2411 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2412 : RISCV::PseudoVMSLT_VX_##suffix; \
2413 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
2414 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
2415 VMSGTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix##_MASK \
2416 : RISCV::PseudoVMSGT_VX_##suffix##_MASK; \
2417 break;
2418 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2419 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2420 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2421 CASE_VMSLT_OPCODES(LMUL_1, M1)
2422 CASE_VMSLT_OPCODES(LMUL_2, M2)
2423 CASE_VMSLT_OPCODES(LMUL_4, M4)
2424 CASE_VMSLT_OPCODES(LMUL_8, M8)
2425#undef CASE_VMSLT_OPCODES
2426 }
2427 // Mask operations use the LMUL from the mask type.
2428 switch (RISCVTargetLowering::getLMUL(VT)) {
2429 default:
2430 llvm_unreachable("Unexpected LMUL!");
2431#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
2432 case RISCVVType::lmulenum: \
2433 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
2434 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
2435 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
2436 break;
2437 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, B64)
2438 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, B32)
2439 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, B16)
2440 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_1, B8)
2441 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_2, B4)
2442 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_4, B2)
2443 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_8, B1)
2444#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
2445 }
2446 SDValue SEW = CurDAG->getTargetConstant(
2447 Val: Log2_32(Value: Src1VT.getScalarSizeInBits()), DL, VT: XLenVT);
2448 SDValue MaskSEW = CurDAG->getTargetConstant(Val: 0, DL, VT: XLenVT);
2449 SDValue VL;
2450 selectVLOp(N: Node->getOperand(Num: 5), VL);
2451 SDValue MaskedOff = Node->getOperand(Num: 1);
2452 SDValue Mask = Node->getOperand(Num: 4);
2453
2454 // If vmsge(u) with minimum value, expand it to vmor mask, maskedoff.
2455 if (IsCmpMinimum) {
2456 // We don't need vmor if the MaskedOff and the Mask are the same
2457 // value.
2458 if (Mask == MaskedOff) {
2459 ReplaceUses(F: Node, T: Mask.getNode());
2460 return;
2461 }
2462 ReplaceNode(F: Node,
2463 T: CurDAG->getMachineNode(Opcode: VMOROpcode, dl: DL, VT,
2464 Ops: {Mask, MaskedOff, VL, MaskSEW}));
2465 return;
2466 }
2467
2468 // If the MaskedOff value and the Mask are the same value use
2469 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
2470 // This avoids needing to copy v0 to vd before starting the next sequence.
2471 if (Mask == MaskedOff) {
2472 SDValue Cmp = SDValue(
2473 CurDAG->getMachineNode(Opcode: VMSLTOpcode, dl: DL, VT, Ops: {Src1, Src2, VL, SEW}),
2474 0);
2475 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMANDNOpcode, dl: DL, VT,
2476 Ops: {Mask, Cmp, VL, MaskSEW}));
2477 return;
2478 }
2479
2480 SDValue PolicyOp =
2481 CurDAG->getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
2482
2483 if (IsCmpConstant) {
2484 SDValue Imm =
2485 selectImm(CurDAG, DL: SDLoc(Src2), VT: XLenVT, Imm: CVal - 1, Subtarget: *Subtarget);
2486
2487 ReplaceNode(F: Node, T: CurDAG->getMachineNode(
2488 Opcode: VMSGTMaskOpcode, dl: DL, VT,
2489 Ops: {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp}));
2490 return;
2491 }
2492
2493 // Otherwise use
2494 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
2495 // The result is mask undisturbed.
2496 // We use the same instructions to emulate mask agnostic behavior, because
2497 // the agnostic result can be either undisturbed or all 1.
2498 SDValue Cmp = SDValue(CurDAG->getMachineNode(Opcode: VMSLTMaskOpcode, dl: DL, VT,
2499 Ops: {MaskedOff, Src1, Src2, Mask,
2500 VL, SEW, PolicyOp}),
2501 0);
2502 // vmxor.mm vd, vd, v0 is used to update active value.
2503 ReplaceNode(F: Node, T: CurDAG->getMachineNode(Opcode: VMXOROpcode, dl: DL, VT,
2504 Ops: {Cmp, Mask, VL, MaskSEW}));
2505 return;
2506 }
2507 case Intrinsic::riscv_vsetvli:
2508 case Intrinsic::riscv_vsetvlimax:
2509 return selectVSETVLI(Node);
2510 case Intrinsic::riscv_sf_vsettnt:
2511 case Intrinsic::riscv_sf_vsettm:
2512 case Intrinsic::riscv_sf_vsettk:
2513 return selectXSfmmVSET(Node);
2514 }
2515 break;
2516 }
2517 case ISD::INTRINSIC_W_CHAIN: {
2518 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2519 switch (IntNo) {
2520 // By default we do not custom select any intrinsic.
2521 default:
2522 break;
2523 case Intrinsic::riscv_vlseg2:
2524 case Intrinsic::riscv_vlseg3:
2525 case Intrinsic::riscv_vlseg4:
2526 case Intrinsic::riscv_vlseg5:
2527 case Intrinsic::riscv_vlseg6:
2528 case Intrinsic::riscv_vlseg7:
2529 case Intrinsic::riscv_vlseg8: {
2530 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2531 /*IsStrided*/ false);
2532 return;
2533 }
2534 case Intrinsic::riscv_vlseg2_mask:
2535 case Intrinsic::riscv_vlseg3_mask:
2536 case Intrinsic::riscv_vlseg4_mask:
2537 case Intrinsic::riscv_vlseg5_mask:
2538 case Intrinsic::riscv_vlseg6_mask:
2539 case Intrinsic::riscv_vlseg7_mask:
2540 case Intrinsic::riscv_vlseg8_mask: {
2541 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2542 /*IsStrided*/ false);
2543 return;
2544 }
2545 case Intrinsic::riscv_vlsseg2:
2546 case Intrinsic::riscv_vlsseg3:
2547 case Intrinsic::riscv_vlsseg4:
2548 case Intrinsic::riscv_vlsseg5:
2549 case Intrinsic::riscv_vlsseg6:
2550 case Intrinsic::riscv_vlsseg7:
2551 case Intrinsic::riscv_vlsseg8: {
2552 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2553 /*IsStrided*/ true);
2554 return;
2555 }
2556 case Intrinsic::riscv_vlsseg2_mask:
2557 case Intrinsic::riscv_vlsseg3_mask:
2558 case Intrinsic::riscv_vlsseg4_mask:
2559 case Intrinsic::riscv_vlsseg5_mask:
2560 case Intrinsic::riscv_vlsseg6_mask:
2561 case Intrinsic::riscv_vlsseg7_mask:
2562 case Intrinsic::riscv_vlsseg8_mask: {
2563 selectVLSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2564 /*IsStrided*/ true);
2565 return;
2566 }
2567 case Intrinsic::riscv_vloxseg2:
2568 case Intrinsic::riscv_vloxseg3:
2569 case Intrinsic::riscv_vloxseg4:
2570 case Intrinsic::riscv_vloxseg5:
2571 case Intrinsic::riscv_vloxseg6:
2572 case Intrinsic::riscv_vloxseg7:
2573 case Intrinsic::riscv_vloxseg8:
2574 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2575 /*IsOrdered*/ true);
2576 return;
2577 case Intrinsic::riscv_vluxseg2:
2578 case Intrinsic::riscv_vluxseg3:
2579 case Intrinsic::riscv_vluxseg4:
2580 case Intrinsic::riscv_vluxseg5:
2581 case Intrinsic::riscv_vluxseg6:
2582 case Intrinsic::riscv_vluxseg7:
2583 case Intrinsic::riscv_vluxseg8:
2584 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2585 /*IsOrdered*/ false);
2586 return;
2587 case Intrinsic::riscv_vloxseg2_mask:
2588 case Intrinsic::riscv_vloxseg3_mask:
2589 case Intrinsic::riscv_vloxseg4_mask:
2590 case Intrinsic::riscv_vloxseg5_mask:
2591 case Intrinsic::riscv_vloxseg6_mask:
2592 case Intrinsic::riscv_vloxseg7_mask:
2593 case Intrinsic::riscv_vloxseg8_mask:
2594 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2595 /*IsOrdered*/ true);
2596 return;
2597 case Intrinsic::riscv_vluxseg2_mask:
2598 case Intrinsic::riscv_vluxseg3_mask:
2599 case Intrinsic::riscv_vluxseg4_mask:
2600 case Intrinsic::riscv_vluxseg5_mask:
2601 case Intrinsic::riscv_vluxseg6_mask:
2602 case Intrinsic::riscv_vluxseg7_mask:
2603 case Intrinsic::riscv_vluxseg8_mask:
2604 selectVLXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2605 /*IsOrdered*/ false);
2606 return;
2607 case Intrinsic::riscv_vlseg8ff:
2608 case Intrinsic::riscv_vlseg7ff:
2609 case Intrinsic::riscv_vlseg6ff:
2610 case Intrinsic::riscv_vlseg5ff:
2611 case Intrinsic::riscv_vlseg4ff:
2612 case Intrinsic::riscv_vlseg3ff:
2613 case Intrinsic::riscv_vlseg2ff: {
2614 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false);
2615 return;
2616 }
2617 case Intrinsic::riscv_vlseg8ff_mask:
2618 case Intrinsic::riscv_vlseg7ff_mask:
2619 case Intrinsic::riscv_vlseg6ff_mask:
2620 case Intrinsic::riscv_vlseg5ff_mask:
2621 case Intrinsic::riscv_vlseg4ff_mask:
2622 case Intrinsic::riscv_vlseg3ff_mask:
2623 case Intrinsic::riscv_vlseg2ff_mask: {
2624 selectVLSEGFF(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true);
2625 return;
2626 }
2627 case Intrinsic::riscv_vloxei:
2628 case Intrinsic::riscv_vloxei_mask:
2629 case Intrinsic::riscv_vluxei:
2630 case Intrinsic::riscv_vluxei_mask: {
2631 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
2632 IntNo == Intrinsic::riscv_vluxei_mask;
2633 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
2634 IntNo == Intrinsic::riscv_vloxei_mask;
2635
2636 MVT VT = Node->getSimpleValueType(ResNo: 0);
2637 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2638
2639 unsigned CurOp = 2;
2640 SmallVector<SDValue, 8> Operands;
2641 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2642
2643 MVT IndexVT;
2644 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2645 /*IsStridedOrIndexed*/ true, Operands,
2646 /*IsLoad=*/true, IndexVT: &IndexVT);
2647
2648 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2649 "Element count mismatch");
2650
2651 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2652 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2653 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2654 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2655 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2656 "index values when XLEN=32");
2657 }
2658 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
2659 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW, LMUL: static_cast<unsigned>(LMUL),
2660 IndexLMUL: static_cast<unsigned>(IndexLMUL));
2661 MachineSDNode *Load =
2662 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2663
2664 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2665
2666 ReplaceNode(F: Node, T: Load);
2667 return;
2668 }
2669 case Intrinsic::riscv_vlm:
2670 case Intrinsic::riscv_vle:
2671 case Intrinsic::riscv_vle_mask:
2672 case Intrinsic::riscv_vlse:
2673 case Intrinsic::riscv_vlse_mask: {
2674 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
2675 IntNo == Intrinsic::riscv_vlse_mask;
2676 bool IsStrided =
2677 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
2678
2679 MVT VT = Node->getSimpleValueType(ResNo: 0);
2680 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2681
2682 // The riscv_vlm intrinsic are always tail agnostic and no passthru
2683 // operand at the IR level. In pseudos, they have both policy and
2684 // passthru operand. The passthru operand is needed to track the
2685 // "tail undefined" state, and the policy is there just for
2686 // for consistency - it will always be "don't care" for the
2687 // unmasked form.
2688 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
2689 unsigned CurOp = 2;
2690 SmallVector<SDValue, 8> Operands;
2691 if (HasPassthruOperand)
2692 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2693 else {
2694 // We eagerly lower to implicit_def (instead of undef), as we
2695 // otherwise fail to select nodes such as: nxv1i1 = undef
2696 SDNode *Passthru =
2697 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT);
2698 Operands.push_back(Elt: SDValue(Passthru, 0));
2699 }
2700 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2701 Operands, /*IsLoad=*/true);
2702
2703 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2704 const RISCV::VLEPseudo *P =
2705 RISCV::getVLEPseudo(Masked: IsMasked, Strided: IsStrided, /*FF*/ false, Log2SEW,
2706 LMUL: static_cast<unsigned>(LMUL));
2707 MachineSDNode *Load =
2708 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2709
2710 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2711
2712 ReplaceNode(F: Node, T: Load);
2713 return;
2714 }
2715 case Intrinsic::riscv_vleff:
2716 case Intrinsic::riscv_vleff_mask: {
2717 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
2718
2719 MVT VT = Node->getSimpleValueType(ResNo: 0);
2720 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2721
2722 unsigned CurOp = 2;
2723 SmallVector<SDValue, 7> Operands;
2724 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2725 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2726 /*IsStridedOrIndexed*/ false, Operands,
2727 /*IsLoad=*/true);
2728
2729 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2730 const RISCV::VLEPseudo *P =
2731 RISCV::getVLEPseudo(Masked: IsMasked, /*Strided*/ false, /*FF*/ true,
2732 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2733 MachineSDNode *Load = CurDAG->getMachineNode(
2734 Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2735 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2736
2737 ReplaceNode(F: Node, T: Load);
2738 return;
2739 }
2740 case Intrinsic::riscv_nds_vln:
2741 case Intrinsic::riscv_nds_vln_mask:
2742 case Intrinsic::riscv_nds_vlnu:
2743 case Intrinsic::riscv_nds_vlnu_mask: {
2744 bool IsMasked = IntNo == Intrinsic::riscv_nds_vln_mask ||
2745 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2746 bool IsUnsigned = IntNo == Intrinsic::riscv_nds_vlnu ||
2747 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2748
2749 MVT VT = Node->getSimpleValueType(ResNo: 0);
2750 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2751 unsigned CurOp = 2;
2752 SmallVector<SDValue, 8> Operands;
2753
2754 Operands.push_back(Elt: Node->getOperand(Num: CurOp++));
2755 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2756 /*IsStridedOrIndexed=*/false, Operands,
2757 /*IsLoad=*/true);
2758
2759 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2760 const RISCV::NDSVLNPseudo *P = RISCV::getNDSVLNPseudo(
2761 Masked: IsMasked, Unsigned: IsUnsigned, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2762 MachineSDNode *Load =
2763 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2764
2765 if (auto *MemOp = dyn_cast<MemSDNode>(Val: Node))
2766 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {MemOp->getMemOperand()});
2767
2768 ReplaceNode(F: Node, T: Load);
2769 return;
2770 }
2771 }
2772 break;
2773 }
2774 case ISD::INTRINSIC_VOID: {
2775 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
2776 switch (IntNo) {
2777 case Intrinsic::riscv_vsseg2:
2778 case Intrinsic::riscv_vsseg3:
2779 case Intrinsic::riscv_vsseg4:
2780 case Intrinsic::riscv_vsseg5:
2781 case Intrinsic::riscv_vsseg6:
2782 case Intrinsic::riscv_vsseg7:
2783 case Intrinsic::riscv_vsseg8: {
2784 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2785 /*IsStrided*/ false);
2786 return;
2787 }
2788 case Intrinsic::riscv_vsseg2_mask:
2789 case Intrinsic::riscv_vsseg3_mask:
2790 case Intrinsic::riscv_vsseg4_mask:
2791 case Intrinsic::riscv_vsseg5_mask:
2792 case Intrinsic::riscv_vsseg6_mask:
2793 case Intrinsic::riscv_vsseg7_mask:
2794 case Intrinsic::riscv_vsseg8_mask: {
2795 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2796 /*IsStrided*/ false);
2797 return;
2798 }
2799 case Intrinsic::riscv_vssseg2:
2800 case Intrinsic::riscv_vssseg3:
2801 case Intrinsic::riscv_vssseg4:
2802 case Intrinsic::riscv_vssseg5:
2803 case Intrinsic::riscv_vssseg6:
2804 case Intrinsic::riscv_vssseg7:
2805 case Intrinsic::riscv_vssseg8: {
2806 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2807 /*IsStrided*/ true);
2808 return;
2809 }
2810 case Intrinsic::riscv_vssseg2_mask:
2811 case Intrinsic::riscv_vssseg3_mask:
2812 case Intrinsic::riscv_vssseg4_mask:
2813 case Intrinsic::riscv_vssseg5_mask:
2814 case Intrinsic::riscv_vssseg6_mask:
2815 case Intrinsic::riscv_vssseg7_mask:
2816 case Intrinsic::riscv_vssseg8_mask: {
2817 selectVSSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2818 /*IsStrided*/ true);
2819 return;
2820 }
2821 case Intrinsic::riscv_vsoxseg2:
2822 case Intrinsic::riscv_vsoxseg3:
2823 case Intrinsic::riscv_vsoxseg4:
2824 case Intrinsic::riscv_vsoxseg5:
2825 case Intrinsic::riscv_vsoxseg6:
2826 case Intrinsic::riscv_vsoxseg7:
2827 case Intrinsic::riscv_vsoxseg8:
2828 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2829 /*IsOrdered*/ true);
2830 return;
2831 case Intrinsic::riscv_vsuxseg2:
2832 case Intrinsic::riscv_vsuxseg3:
2833 case Intrinsic::riscv_vsuxseg4:
2834 case Intrinsic::riscv_vsuxseg5:
2835 case Intrinsic::riscv_vsuxseg6:
2836 case Intrinsic::riscv_vsuxseg7:
2837 case Intrinsic::riscv_vsuxseg8:
2838 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ false,
2839 /*IsOrdered*/ false);
2840 return;
2841 case Intrinsic::riscv_vsoxseg2_mask:
2842 case Intrinsic::riscv_vsoxseg3_mask:
2843 case Intrinsic::riscv_vsoxseg4_mask:
2844 case Intrinsic::riscv_vsoxseg5_mask:
2845 case Intrinsic::riscv_vsoxseg6_mask:
2846 case Intrinsic::riscv_vsoxseg7_mask:
2847 case Intrinsic::riscv_vsoxseg8_mask:
2848 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2849 /*IsOrdered*/ true);
2850 return;
2851 case Intrinsic::riscv_vsuxseg2_mask:
2852 case Intrinsic::riscv_vsuxseg3_mask:
2853 case Intrinsic::riscv_vsuxseg4_mask:
2854 case Intrinsic::riscv_vsuxseg5_mask:
2855 case Intrinsic::riscv_vsuxseg6_mask:
2856 case Intrinsic::riscv_vsuxseg7_mask:
2857 case Intrinsic::riscv_vsuxseg8_mask:
2858 selectVSXSEG(Node, NF: getSegInstNF(Intrinsic: IntNo), /*IsMasked*/ true,
2859 /*IsOrdered*/ false);
2860 return;
2861 case Intrinsic::riscv_vsoxei:
2862 case Intrinsic::riscv_vsoxei_mask:
2863 case Intrinsic::riscv_vsuxei:
2864 case Intrinsic::riscv_vsuxei_mask: {
2865 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
2866 IntNo == Intrinsic::riscv_vsuxei_mask;
2867 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
2868 IntNo == Intrinsic::riscv_vsoxei_mask;
2869
2870 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2871 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2872
2873 unsigned CurOp = 2;
2874 SmallVector<SDValue, 8> Operands;
2875 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2876
2877 MVT IndexVT;
2878 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2879 /*IsStridedOrIndexed*/ true, Operands,
2880 /*IsLoad=*/false, IndexVT: &IndexVT);
2881
2882 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
2883 "Element count mismatch");
2884
2885 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2886 RISCVVType::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(VT: IndexVT);
2887 unsigned IndexLog2EEW = Log2_32(Value: IndexVT.getScalarSizeInBits());
2888 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2889 reportFatalUsageError(reason: "The V extension does not support EEW=64 for "
2890 "index values when XLEN=32");
2891 }
2892 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
2893 Masked: IsMasked, Ordered: IsOrdered, Log2SEW: IndexLog2EEW,
2894 LMUL: static_cast<unsigned>(LMUL), IndexLMUL: static_cast<unsigned>(IndexLMUL));
2895 MachineSDNode *Store =
2896 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2897
2898 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2899
2900 ReplaceNode(F: Node, T: Store);
2901 return;
2902 }
2903 case Intrinsic::riscv_vsm:
2904 case Intrinsic::riscv_vse:
2905 case Intrinsic::riscv_vse_mask:
2906 case Intrinsic::riscv_vsse:
2907 case Intrinsic::riscv_vsse_mask: {
2908 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
2909 IntNo == Intrinsic::riscv_vsse_mask;
2910 bool IsStrided =
2911 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
2912
2913 MVT VT = Node->getOperand(Num: 2)->getSimpleValueType(ResNo: 0);
2914 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
2915
2916 unsigned CurOp = 2;
2917 SmallVector<SDValue, 8> Operands;
2918 Operands.push_back(Elt: Node->getOperand(Num: CurOp++)); // Store value.
2919
2920 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStridedOrIndexed: IsStrided,
2921 Operands);
2922
2923 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
2924 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
2925 Masked: IsMasked, Strided: IsStrided, Log2SEW, LMUL: static_cast<unsigned>(LMUL));
2926 MachineSDNode *Store =
2927 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2928 CurDAG->setNodeMemRefs(N: Store, NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2929
2930 ReplaceNode(F: Node, T: Store);
2931 return;
2932 }
2933 case Intrinsic::riscv_sf_vc_x_se:
2934 case Intrinsic::riscv_sf_vc_i_se:
2935 selectSF_VC_X_SE(Node);
2936 return;
2937 case Intrinsic::riscv_sf_vlte8:
2938 case Intrinsic::riscv_sf_vlte16:
2939 case Intrinsic::riscv_sf_vlte32:
2940 case Intrinsic::riscv_sf_vlte64: {
2941 unsigned Log2SEW;
2942 unsigned PseudoInst;
2943 switch (IntNo) {
2944 case Intrinsic::riscv_sf_vlte8:
2945 PseudoInst = RISCV::PseudoSF_VLTE8;
2946 Log2SEW = 3;
2947 break;
2948 case Intrinsic::riscv_sf_vlte16:
2949 PseudoInst = RISCV::PseudoSF_VLTE16;
2950 Log2SEW = 4;
2951 break;
2952 case Intrinsic::riscv_sf_vlte32:
2953 PseudoInst = RISCV::PseudoSF_VLTE32;
2954 Log2SEW = 5;
2955 break;
2956 case Intrinsic::riscv_sf_vlte64:
2957 PseudoInst = RISCV::PseudoSF_VLTE64;
2958 Log2SEW = 6;
2959 break;
2960 }
2961
2962 SDValue SEWOp = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
2963 SDValue TWidenOp = CurDAG->getTargetConstant(Val: 1, DL, VT: XLenVT);
2964 SDValue Operands[] = {Node->getOperand(Num: 2),
2965 Node->getOperand(Num: 3),
2966 Node->getOperand(Num: 4),
2967 SEWOp,
2968 TWidenOp,
2969 Node->getOperand(Num: 0)};
2970
2971 MachineSDNode *TileLoad =
2972 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
2973 CurDAG->setNodeMemRefs(N: TileLoad,
2974 NewMemRefs: {cast<MemSDNode>(Val: Node)->getMemOperand()});
2975
2976 ReplaceNode(F: Node, T: TileLoad);
2977 return;
2978 }
2979 case Intrinsic::riscv_sf_mm_s_s:
2980 case Intrinsic::riscv_sf_mm_s_u:
2981 case Intrinsic::riscv_sf_mm_u_s:
2982 case Intrinsic::riscv_sf_mm_u_u:
2983 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
2984 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
2985 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
2986 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
2987 case Intrinsic::riscv_sf_mm_f_f: {
2988 bool HasFRM = false;
2989 unsigned PseudoInst;
2990 switch (IntNo) {
2991 case Intrinsic::riscv_sf_mm_s_s:
2992 PseudoInst = RISCV::PseudoSF_MM_S_S;
2993 break;
2994 case Intrinsic::riscv_sf_mm_s_u:
2995 PseudoInst = RISCV::PseudoSF_MM_S_U;
2996 break;
2997 case Intrinsic::riscv_sf_mm_u_s:
2998 PseudoInst = RISCV::PseudoSF_MM_U_S;
2999 break;
3000 case Intrinsic::riscv_sf_mm_u_u:
3001 PseudoInst = RISCV::PseudoSF_MM_U_U;
3002 break;
3003 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
3004 PseudoInst = RISCV::PseudoSF_MM_E5M2_E5M2;
3005 HasFRM = true;
3006 break;
3007 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
3008 PseudoInst = RISCV::PseudoSF_MM_E5M2_E4M3;
3009 HasFRM = true;
3010 break;
3011 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
3012 PseudoInst = RISCV::PseudoSF_MM_E4M3_E5M2;
3013 HasFRM = true;
3014 break;
3015 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
3016 PseudoInst = RISCV::PseudoSF_MM_E4M3_E4M3;
3017 HasFRM = true;
3018 break;
3019 case Intrinsic::riscv_sf_mm_f_f:
3020 if (Node->getOperand(Num: 3).getValueType().getScalarType() == MVT::bf16)
3021 PseudoInst = RISCV::PseudoSF_MM_F_F_ALT;
3022 else
3023 PseudoInst = RISCV::PseudoSF_MM_F_F;
3024 HasFRM = true;
3025 break;
3026 }
3027 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3028 SDValue Op1 = Node->getOperand(Num: 3);
3029 SDValue Op2 = Node->getOperand(Num: 4);
3030 MVT VT = Op1->getSimpleValueType(ResNo: 0);
3031 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3032 SDValue TmOp = Node->getOperand(Num: 5);
3033 SDValue TnOp = Node->getOperand(Num: 6);
3034 SDValue TkOp = Node->getOperand(Num: 7);
3035 SDValue TWidenOp = Node->getOperand(Num: 8);
3036 SDValue Chain = Node->getOperand(Num: 0);
3037
3038 // sf.mm.f.f with sew=32, twiden=2 is invalid
3039 if (IntNo == Intrinsic::riscv_sf_mm_f_f && Log2SEW == 5 &&
3040 TWidenOp->getAsZExtVal() == 2)
3041 reportFatalUsageError(reason: "sf.mm.f.f doesn't support (sew=32, twiden=2)");
3042
3043 SmallVector<SDValue, 10> Operands(
3044 {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Op1, Op2});
3045 if (HasFRM)
3046 Operands.push_back(
3047 Elt: CurDAG->getTargetConstant(Val: RISCVFPRndMode::DYN, DL, VT: XLenVT));
3048 Operands.append(IL: {TmOp, TnOp, TkOp,
3049 CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT), TWidenOp,
3050 Chain});
3051
3052 auto *NewNode =
3053 CurDAG->getMachineNode(Opcode: PseudoInst, dl: DL, VTs: Node->getVTList(), Ops: Operands);
3054
3055 ReplaceNode(F: Node, T: NewNode);
3056 return;
3057 }
3058 case Intrinsic::riscv_sf_vtzero_t: {
3059 uint64_t TileNum = Node->getConstantOperandVal(Num: 2);
3060 SDValue Tm = Node->getOperand(Num: 3);
3061 SDValue Tn = Node->getOperand(Num: 4);
3062 SDValue Log2SEW = Node->getOperand(Num: 5);
3063 SDValue TWiden = Node->getOperand(Num: 6);
3064 SDValue Chain = Node->getOperand(Num: 0);
3065 auto *NewNode = CurDAG->getMachineNode(
3066 Opcode: RISCV::PseudoSF_VTZERO_T, dl: DL, VTs: Node->getVTList(),
3067 Ops: {CurDAG->getRegister(Reg: getTileReg(TileNum), VT: XLenVT), Tm, Tn, Log2SEW,
3068 TWiden, Chain});
3069
3070 ReplaceNode(F: Node, T: NewNode);
3071 return;
3072 }
3073 }
3074 break;
3075 }
3076 case ISD::BITCAST: {
3077 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3078 // Just drop bitcasts between vectors if both are fixed or both are
3079 // scalable.
3080 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
3081 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
3082 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3083 CurDAG->RemoveDeadNode(N: Node);
3084 return;
3085 }
3086 if (Subtarget->hasStdExtP()) {
3087 bool Is32BitCast =
3088 (VT == MVT::i32 && (SrcVT == MVT::v4i8 || SrcVT == MVT::v2i16)) ||
3089 (SrcVT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
3090 bool Is64BitCast =
3091 (VT == MVT::i64 && (SrcVT == MVT::v8i8 || SrcVT == MVT::v4i16 ||
3092 SrcVT == MVT::v2i32)) ||
3093 (SrcVT == MVT::i64 &&
3094 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
3095 if (Is32BitCast || Is64BitCast) {
3096 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3097 CurDAG->RemoveDeadNode(N: Node);
3098 return;
3099 }
3100 }
3101 break;
3102 }
3103 case ISD::SPLAT_VECTOR: {
3104 if (!Subtarget->hasStdExtP())
3105 break;
3106 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 0))) {
3107 bool IsDoubleWide = Subtarget->isPExtPackedDoubleType(VT);
3108
3109 if (ConstNode->isZero()) {
3110 MCPhysReg X0Reg = IsDoubleWide ? RISCV::X0_Pair : RISCV::X0;
3111 SDValue New =
3112 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: X0Reg, VT);
3113 ReplaceNode(F: Node, T: New.getNode());
3114 return;
3115 }
3116
3117 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3118 APInt Val = ConstNode->getAPIntValue().trunc(width: EltSize);
3119
3120 // Use LI for all ones since it can be compressed to c.li.
3121 if (Val.isAllOnes() && !IsDoubleWide) {
3122 SDNode *NewNode = CurDAG->getMachineNode(
3123 Opcode: RISCV::ADDI, dl: DL, VT, Op1: CurDAG->getRegister(Reg: RISCV::X0, VT),
3124 Op2: CurDAG->getAllOnesConstant(DL, VT: XLenVT, /*IsTarget=*/true));
3125 ReplaceNode(F: Node, T: NewNode);
3126 return;
3127 }
3128
3129 // Find the smallest splat.
3130 if (Val.getBitWidth() > 16 && Val.isSplat(SplatSizeInBits: 16))
3131 Val = Val.trunc(width: 16);
3132 if (Val.getBitWidth() > 8 && Val.isSplat(SplatSizeInBits: 8))
3133 Val = Val.trunc(width: 8);
3134
3135 EltSize = Val.getBitWidth();
3136 int64_t Imm = Val.getSExtValue();
3137
3138 unsigned Opc = 0;
3139 if (EltSize == 8) {
3140 Opc = IsDoubleWide ? RISCV::PLI_DB : RISCV::PLI_B;
3141 } else if (EltSize == 16 && isInt<10>(x: Imm)) {
3142 Opc = IsDoubleWide ? RISCV::PLI_DH : RISCV::PLI_H;
3143 } else if (!IsDoubleWide && EltSize == 32 && isInt<10>(x: Imm)) {
3144 Opc = RISCV::PLI_W;
3145 } else if (EltSize == 16 && isShiftedInt<10, 6>(x: Imm)) {
3146 Opc = IsDoubleWide ? RISCV::PLUI_DH : RISCV::PLUI_H;
3147 Imm = Imm >> 6;
3148 } else if (!IsDoubleWide && EltSize == 32 && isShiftedInt<10, 22>(x: Imm)) {
3149 Opc = RISCV::PLUI_W;
3150 Imm = Imm >> 22;
3151 }
3152
3153 if (Opc) {
3154 SDNode *NewNode = CurDAG->getMachineNode(
3155 Opcode: Opc, dl: DL, VT, Op1: CurDAG->getSignedTargetConstant(Val: Imm, DL, VT: XLenVT));
3156 ReplaceNode(F: Node, T: NewNode);
3157 return;
3158 }
3159 }
3160
3161 break;
3162 }
3163 case ISD::SCALAR_TO_VECTOR:
3164 if (Subtarget->hasStdExtP()) {
3165 MVT SrcVT = Node->getOperand(Num: 0).getSimpleValueType();
3166 if ((VT == MVT::v2i32 && SrcVT == MVT::i64) ||
3167 (VT == MVT::v4i8 && SrcVT == MVT::i32)) {
3168 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3169 CurDAG->RemoveDeadNode(N: Node);
3170 return;
3171 }
3172 }
3173 break;
3174 case ISD::INSERT_SUBVECTOR:
3175 case RISCVISD::TUPLE_INSERT: {
3176 SDValue V = Node->getOperand(Num: 0);
3177 SDValue SubV = Node->getOperand(Num: 1);
3178 SDLoc DL(SubV);
3179 auto Idx = Node->getConstantOperandVal(Num: 2);
3180 MVT SubVecVT = SubV.getSimpleValueType();
3181
3182 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3183 MVT SubVecContainerVT = SubVecVT;
3184 // Establish the correct scalable-vector types for any fixed-length type.
3185 if (SubVecVT.isFixedLengthVector()) {
3186 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT: SubVecVT);
3187 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
3188 [[maybe_unused]] bool ExactlyVecRegSized =
3189 Subtarget->expandVScale(X: SubVecVT.getSizeInBits())
3190 .isKnownMultipleOf(RHS: Subtarget->expandVScale(X: VecRegSize));
3191 assert(isPowerOf2_64(Subtarget->expandVScale(SubVecVT.getSizeInBits())
3192 .getKnownMinValue()));
3193 assert(Idx == 0 && (ExactlyVecRegSized || V.isUndef()));
3194 }
3195 MVT ContainerVT = VT;
3196 if (VT.isFixedLengthVector())
3197 ContainerVT = TLI.getContainerForFixedLengthVector(VT);
3198
3199 const auto *TRI = Subtarget->getRegisterInfo();
3200 unsigned SubRegIdx;
3201 std::tie(args&: SubRegIdx, args&: Idx) =
3202 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3203 VecVT: ContainerVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3204
3205 // If the Idx hasn't been completely eliminated then this is a subvector
3206 // insert which doesn't naturally align to a vector register. These must
3207 // be handled using instructions to manipulate the vector registers.
3208 if (Idx != 0)
3209 break;
3210
3211 RISCVVType::VLMUL SubVecLMUL =
3212 RISCVTargetLowering::getLMUL(VT: SubVecContainerVT);
3213 [[maybe_unused]] bool IsSubVecPartReg =
3214 SubVecLMUL == RISCVVType::VLMUL::LMUL_F2 ||
3215 SubVecLMUL == RISCVVType::VLMUL::LMUL_F4 ||
3216 SubVecLMUL == RISCVVType::VLMUL::LMUL_F8;
3217 assert((V.getValueType().isRISCVVectorTuple() || !IsSubVecPartReg ||
3218 V.isUndef()) &&
3219 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
3220 "the subvector is smaller than a full-sized register");
3221
3222 // If we haven't set a SubRegIdx, then we must be going between
3223 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
3224 if (SubRegIdx == RISCV::NoSubRegister) {
3225 unsigned InRegClassID =
3226 RISCVTargetLowering::getRegClassIDForVecVT(VT: ContainerVT);
3227 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3228 InRegClassID &&
3229 "Unexpected subvector extraction");
3230 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3231 SDNode *NewNode = CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
3232 dl: DL, VT, Op1: SubV, Op2: RC);
3233 ReplaceNode(F: Node, T: NewNode);
3234 return;
3235 }
3236
3237 SDValue Insert = CurDAG->getTargetInsertSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V, Subreg: SubV);
3238 ReplaceNode(F: Node, T: Insert.getNode());
3239 return;
3240 }
3241 case RISCVISD::TUPLE_CAST: {
3242 // TUPLE_CAST reinterprets a vector tuple as a different tuple type with
3243 // the same register class (same NF and LMUL), just a different minimum
3244 // number of elements per field. The two types occupy identical
3245 // registers, so just forward the operand like we do for same-register-
3246 // class ISD::BITCAST, without emitting a copy.
3247 assert(RISCVTargetLowering::getRegClassIDForVecVT(VT) ==
3248 RISCVTargetLowering::getRegClassIDForVecVT(
3249 Node->getOperand(0).getSimpleValueType()) &&
3250 "Expected input and output of TUPLE_CAST to use the same "
3251 "register class");
3252 ReplaceUses(F: SDValue(Node, 0), T: Node->getOperand(Num: 0));
3253 CurDAG->RemoveDeadNode(N: Node);
3254 return;
3255 }
3256 case ISD::EXTRACT_SUBVECTOR:
3257 case RISCVISD::TUPLE_EXTRACT: {
3258 if (Subtarget->hasStdExtP())
3259 break;
3260
3261 SDValue V = Node->getOperand(Num: 0);
3262 auto Idx = Node->getConstantOperandVal(Num: 1);
3263 MVT InVT = V.getSimpleValueType();
3264
3265 SDLoc DL(V);
3266
3267 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3268 MVT SubVecContainerVT = VT;
3269 // Establish the correct scalable-vector types for any fixed-length type.
3270 if (VT.isFixedLengthVector()) {
3271 assert(Idx == 0);
3272 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
3273 }
3274 if (InVT.isFixedLengthVector())
3275 InVT = TLI.getContainerForFixedLengthVector(VT: InVT);
3276
3277 const auto *TRI = Subtarget->getRegisterInfo();
3278 unsigned SubRegIdx;
3279 std::tie(args&: SubRegIdx, args&: Idx) =
3280 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3281 VecVT: InVT, SubVecVT: SubVecContainerVT, InsertExtractIdx: Idx, TRI);
3282
3283 // If the Idx hasn't been completely eliminated then this is a subvector
3284 // extract which doesn't naturally align to a vector register. These must
3285 // be handled using instructions to manipulate the vector registers.
3286 if (Idx != 0)
3287 break;
3288
3289 // If we haven't set a SubRegIdx, then we must be going between
3290 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
3291 if (SubRegIdx == RISCV::NoSubRegister) {
3292 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT: InVT);
3293 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
3294 InRegClassID &&
3295 "Unexpected subvector extraction");
3296 SDValue RC = CurDAG->getTargetConstant(Val: InRegClassID, DL, VT: XLenVT);
3297 SDNode *NewNode =
3298 CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT, Op1: V, Op2: RC);
3299 ReplaceNode(F: Node, T: NewNode);
3300 return;
3301 }
3302
3303 SDValue Extract = CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx, DL, VT, Operand: V);
3304 ReplaceNode(F: Node, T: Extract.getNode());
3305 return;
3306 }
3307 case RISCVISD::VMV_S_X_VL:
3308 case RISCVISD::VFMV_S_F_VL:
3309 case RISCVISD::VMV_V_X_VL:
3310 case RISCVISD::VFMV_V_F_VL: {
3311 // Try to match splat of a scalar load to a strided load with stride of x0.
3312 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
3313 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
3314 if (!Node->getOperand(Num: 0).isUndef())
3315 break;
3316 SDValue Src = Node->getOperand(Num: 1);
3317 auto *Ld = dyn_cast<LoadSDNode>(Val&: Src);
3318 // Can't fold load update node because the second
3319 // output is used so that load update node can't be removed.
3320 if (!Ld || Ld->isIndexed())
3321 break;
3322 EVT MemVT = Ld->getMemoryVT();
3323 // The memory VT should be the same size as the element type.
3324 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
3325 break;
3326 if (!IsProfitableToFold(N: Src, U: Node, Root: Node) ||
3327 !IsLegalToFold(N: Src, U: Node, Root: Node, OptLevel: TM.getOptLevel()))
3328 break;
3329
3330 SDValue VL;
3331 if (IsScalarMove) {
3332 // We could deal with more VL if we update the VSETVLI insert pass to
3333 // avoid introducing more VSETVLI.
3334 if (!isOneConstant(V: Node->getOperand(Num: 2)))
3335 break;
3336 selectVLOp(N: Node->getOperand(Num: 2), VL);
3337 } else
3338 selectVLOp(N: Node->getOperand(Num: 2), VL);
3339
3340 unsigned Log2SEW = Log2_32(Value: VT.getScalarSizeInBits());
3341 SDValue SEW = CurDAG->getTargetConstant(Val: Log2SEW, DL, VT: XLenVT);
3342
3343 // If VL=1, then we don't need to do a strided load and can just do a
3344 // regular load.
3345 bool IsStrided = !isOneConstant(V: VL);
3346
3347 // Only do a strided load if we have optimized zero-stride vector load.
3348 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
3349 break;
3350
3351 SmallVector<SDValue> Operands = {
3352 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT), 0),
3353 Ld->getBasePtr()};
3354 if (IsStrided)
3355 Operands.push_back(Elt: CurDAG->getRegister(Reg: RISCV::X0, VT: XLenVT));
3356 uint64_t Policy = RISCVVType::MASK_AGNOSTIC | RISCVVType::TAIL_AGNOSTIC;
3357 SDValue PolicyOp = CurDAG->getTargetConstant(Val: Policy, DL, VT: XLenVT);
3358 Operands.append(IL: {VL, SEW, PolicyOp, Ld->getChain()});
3359
3360 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
3361 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
3362 /*IsMasked*/ Masked: false, Strided: IsStrided, /*FF*/ false,
3363 Log2SEW, LMUL: static_cast<unsigned>(LMUL));
3364 MachineSDNode *Load =
3365 CurDAG->getMachineNode(Opcode: P->Pseudo, dl: DL, ResultTys: {VT, MVT::Other}, Ops: Operands);
3366 // Update the chain.
3367 ReplaceUses(F: Src.getValue(R: 1), T: SDValue(Load, 1));
3368 // Record the mem-refs
3369 CurDAG->setNodeMemRefs(N: Load, NewMemRefs: {Ld->getMemOperand()});
3370 // Replace the splat with the vlse.
3371 ReplaceNode(F: Node, T: Load);
3372 return;
3373 }
3374 case RISCVISD::LPAD_CALL:
3375 case RISCVISD::LPAD_CALL_INDIRECT: {
3376 bool IsIndirect = Opcode == RISCVISD::LPAD_CALL_INDIRECT;
3377 unsigned PseudoOpc = IsIndirect ? RISCV::PseudoCALLIndirectLpadAlign
3378 : RISCV::PseudoCALLLpadAlign;
3379
3380 uint32_t LpadLabel = 0;
3381 if (std::optional<uint32_t> Label =
3382 Subtarget->getCLOpts().landing_pad_label) {
3383 if (!isUInt<20>(x: *Label))
3384 report_fatal_error(reason: "riscv-landing-pad-label=<val>, <val> needs to fit "
3385 "in unsigned 20-bits");
3386 LpadLabel = *Label;
3387 }
3388
3389 // Preserve the argument-register and register-mask operands, between
3390 // Callee and the optional glue, so the pseudo call still reports its
3391 // call-preserved mask to the register allocator.
3392 SmallVector<SDValue, 8> Ops;
3393 Ops.push_back(Elt: Node->getOperand(Num: 1));
3394 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: LpadLabel, DL, VT: XLenVT));
3395
3396 unsigned NumOps = Node->getNumOperands();
3397 bool HasGlue = Node->getGluedNode() != nullptr;
3398 unsigned RegOperandsEnd = HasGlue ? NumOps - 1 : NumOps;
3399 for (unsigned I = 2; I != RegOperandsEnd; ++I)
3400 Ops.push_back(Elt: Node->getOperand(Num: I));
3401
3402 Ops.push_back(Elt: Node->getOperand(Num: 0));
3403 if (HasGlue)
3404 Ops.push_back(Elt: Node->getOperand(Num: NumOps - 1));
3405
3406 ReplaceNode(F: Node,
3407 T: CurDAG->getMachineNode(Opcode: PseudoOpc, dl: DL, VTs: Node->getVTList(), Ops));
3408 return;
3409 }
3410 case ISD::PREFETCH:
3411 // MIPS's prefetch instruction already encodes the hint within the
3412 // instruction itself, so no extra NTL hint is needed.
3413 if (Subtarget->hasVendorXMIPSCBOP())
3414 break;
3415
3416 unsigned Locality = Node->getConstantOperandVal(Num: 3);
3417 if (Locality > 2)
3418 break;
3419
3420 auto *LoadStoreMem = cast<MemSDNode>(Val: Node);
3421 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
3422 MMO->setFlags(MachineMemOperand::MONonTemporal);
3423
3424 int NontemporalLevel = 0;
3425 switch (Locality) {
3426 case 0:
3427 NontemporalLevel = 3; // NTL.ALL
3428 break;
3429 case 1:
3430 NontemporalLevel = 1; // NTL.PALL
3431 break;
3432 case 2:
3433 NontemporalLevel = 0; // NTL.P1
3434 break;
3435 default:
3436 llvm_unreachable("unexpected locality value.");
3437 }
3438
3439 if (NontemporalLevel & 0b1)
3440 MMO->setFlags(MONontemporalBit0);
3441 if (NontemporalLevel & 0b10)
3442 MMO->setFlags(MONontemporalBit1);
3443 break;
3444 }
3445
3446 // Select the default instruction.
3447 SelectCode(N: Node);
3448}
3449
3450bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
3451 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
3452 std::vector<SDValue> &OutOps) {
3453 // Always produce a register and immediate operand, as expected by
3454 // RISCVAsmPrinter::PrintAsmMemoryOperand.
3455 switch (ConstraintID) {
3456 case InlineAsm::ConstraintCode::o:
3457 case InlineAsm::ConstraintCode::m: {
3458 SDValue Op0, Op1;
3459 [[maybe_unused]] bool Found = SelectAddrRegImm(Addr: Op, Base&: Op0, Offset&: Op1);
3460 assert(Found && "SelectAddrRegImm should always succeed");
3461 OutOps.push_back(x: Op0);
3462 OutOps.push_back(x: Op1);
3463 return false;
3464 }
3465 case InlineAsm::ConstraintCode::A:
3466 OutOps.push_back(x: Op);
3467 OutOps.push_back(
3468 x: CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget->getXLenVT()));
3469 return false;
3470 default:
3471 report_fatal_error(reason: "Unexpected asm memory constraint " +
3472 InlineAsm::getMemConstraintName(C: ConstraintID));
3473 }
3474
3475 return true;
3476}
3477
3478bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
3479 SDValue &Offset) {
3480 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Addr)) {
3481 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT: Subtarget->getXLenVT());
3482 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT: Subtarget->getXLenVT());
3483 return true;
3484 }
3485
3486 return false;
3487}
3488
3489// Fold constant addresses.
3490static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
3491 const MVT VT, const RISCVSubtarget *Subtarget,
3492 SDValue Addr, SDValue &Base, SDValue &Offset,
3493 bool IsPrefetch = false) {
3494 if (!isa<ConstantSDNode>(Val: Addr))
3495 return false;
3496
3497 int64_t CVal = cast<ConstantSDNode>(Val&: Addr)->getSExtValue();
3498
3499 // If the constant is a simm12, we can fold the whole constant and use X0 as
3500 // the base. If the constant can be materialized with LUI+simm12, use LUI as
3501 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
3502 int64_t Lo12 = SignExtend64<12>(x: CVal);
3503 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
3504 if (!Subtarget->is64Bit() || isInt<32>(x: Hi)) {
3505 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3506 return false;
3507 if (Hi) {
3508 int64_t Hi20 = (Hi >> 12) & 0xfffff;
3509 Base = SDValue(
3510 CurDAG->getMachineNode(Opcode: RISCV::LUI, dl: DL, VT,
3511 Op1: CurDAG->getTargetConstant(Val: Hi20, DL, VT)),
3512 0);
3513 } else {
3514 Base = CurDAG->getRegister(Reg: RISCV::X0, VT);
3515 }
3516 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3517 return true;
3518 }
3519
3520 // Ask how constant materialization would handle this constant.
3521 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: CVal, STI: *Subtarget);
3522
3523 // If the last instruction would be an ADDI, we can fold its immediate and
3524 // emit the rest of the sequence as the base.
3525 if (Seq.back().getOpcode() != RISCV::ADDI)
3526 return false;
3527 Lo12 = Seq.back().getImm();
3528 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3529 return false;
3530
3531 // Drop the last instruction.
3532 Seq.pop_back();
3533 assert(!Seq.empty() && "Expected more instructions in sequence");
3534
3535 Base = selectImmSeq(CurDAG, DL, VT, Seq);
3536 Offset = CurDAG->getSignedTargetConstant(Val: Lo12, DL, VT);
3537 return true;
3538}
3539
3540// Is this ADD instruction only used as the base pointer of scalar loads and
3541// stores?
3542static bool isWorthFoldingAdd(SDValue Add) {
3543 for (auto *User : Add->users()) {
3544 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
3545 User->getOpcode() != RISCVISD::LD_RV32 &&
3546 User->getOpcode() != RISCVISD::SD_RV32 &&
3547 User->getOpcode() != ISD::ATOMIC_LOAD &&
3548 User->getOpcode() != ISD::ATOMIC_STORE)
3549 return false;
3550 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3551 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
3552 VT != MVT::f64)
3553 return false;
3554 // Don't allow stores of the value. It must be used as the address.
3555 if (User->getOpcode() == ISD::STORE &&
3556 cast<StoreSDNode>(Val: User)->getValue() == Add)
3557 return false;
3558 if (User->getOpcode() == ISD::ATOMIC_STORE &&
3559 cast<AtomicSDNode>(Val: User)->getVal() == Add)
3560 return false;
3561 if (User->getOpcode() == RISCVISD::SD_RV32 &&
3562 (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add))
3563 return false;
3564 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
3565 return false;
3566 }
3567
3568 return true;
3569}
3570
3571bool isRegImmLoadOrStore(SDNode *User, SDValue Add) {
3572 switch (User->getOpcode()) {
3573 default:
3574 return false;
3575 case ISD::LOAD:
3576 case RISCVISD::LD_RV32:
3577 case ISD::ATOMIC_LOAD:
3578 break;
3579 case ISD::STORE:
3580 // Don't allow stores of Add. It must only be used as the address.
3581 if (cast<StoreSDNode>(Val: User)->getValue() == Add)
3582 return false;
3583 break;
3584 case RISCVISD::SD_RV32:
3585 // Don't allow stores of Add. It must only be used as the address.
3586 if (User->getOperand(Num: 0) == Add || User->getOperand(Num: 1) == Add)
3587 return false;
3588 break;
3589 case ISD::ATOMIC_STORE:
3590 // Don't allow stores of Add. It must only be used as the address.
3591 if (cast<AtomicSDNode>(Val: User)->getVal() == Add)
3592 return false;
3593 break;
3594 }
3595
3596 return true;
3597}
3598
3599// To prevent SelectAddrRegImm from folding offsets that conflict with the
3600// fusion of PseudoMovAddr, check if the offset of every use of a given address
3601// is within the alignment.
3602bool RISCVDAGToDAGISel::areOffsetsWithinAlignment(SDValue Addr,
3603 Align Alignment) {
3604 assert(Addr->getOpcode() == RISCVISD::ADD_LO);
3605 for (auto *User : Addr->users()) {
3606 // If the user is a load or store, then the offset is 0 which is always
3607 // within alignment.
3608 if (isRegImmLoadOrStore(User, Add: Addr))
3609 continue;
3610
3611 if (CurDAG->isBaseWithConstantOffset(Op: SDValue(User, 0))) {
3612 int64_t CVal = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
3613 if (!isInt<12>(x: CVal) || Alignment <= CVal)
3614 return false;
3615
3616 // Make sure all uses are foldable load/stores.
3617 for (auto *AddUser : User->users())
3618 if (!isRegImmLoadOrStore(User: AddUser, Add: SDValue(User, 0)))
3619 return false;
3620
3621 continue;
3622 }
3623
3624 return false;
3625 }
3626
3627 return true;
3628}
3629
3630bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
3631 SDValue &Offset) {
3632 if (SelectAddrFrameIndex(Addr, Base, Offset))
3633 return true;
3634
3635 SDLoc DL(Addr);
3636 MVT VT = Addr.getSimpleValueType();
3637
3638 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
3639 bool CanFold = true;
3640 // Unconditionally fold if operand 1 is not a global address (e.g.
3641 // externsymbol)
3642 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val: Addr.getOperand(i: 1))) {
3643 const DataLayout &DL = CurDAG->getDataLayout();
3644 Align Alignment = commonAlignment(
3645 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3646 if (!areOffsetsWithinAlignment(Addr, Alignment))
3647 CanFold = false;
3648 }
3649 if (CanFold) {
3650 Base = Addr.getOperand(i: 0);
3651 Offset = Addr.getOperand(i: 1);
3652 return true;
3653 }
3654 }
3655
3656 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3657 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3658 if (isInt<12>(x: CVal)) {
3659 Base = Addr.getOperand(i: 0);
3660 if (Base.getOpcode() == RISCVISD::ADD_LO) {
3661 SDValue LoOperand = Base.getOperand(i: 1);
3662 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: LoOperand)) {
3663 // If the Lo in (ADD_LO hi, lo) is a global variable's address
3664 // (its low part, really), then we can rely on the alignment of that
3665 // variable to provide a margin of safety before low part can overflow
3666 // the 12 bits of the load/store offset. Check if CVal falls within
3667 // that margin; if so (low part + CVal) can't overflow.
3668 const DataLayout &DL = CurDAG->getDataLayout();
3669 Align Alignment = commonAlignment(
3670 A: GA->getGlobal()->getPointerAlignment(DL), Offset: GA->getOffset());
3671 if ((CVal == 0 || Alignment > CVal) &&
3672 areOffsetsWithinAlignment(Addr: Base, Alignment)) {
3673 int64_t CombinedOffset = CVal + GA->getOffset();
3674 Base = Base.getOperand(i: 0);
3675 Offset = CurDAG->getTargetGlobalAddress(
3676 GV: GA->getGlobal(), DL: SDLoc(LoOperand), VT: LoOperand.getValueType(),
3677 offset: CombinedOffset, TargetFlags: GA->getTargetFlags());
3678 return true;
3679 }
3680 }
3681 }
3682
3683 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3684 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3685 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3686 return true;
3687 }
3688 }
3689
3690 // Handle ADD with large immediates.
3691 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3692 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3693 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3694
3695 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
3696 // an ADDI for part of the offset and fold the rest into the load/store.
3697 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
3698 if (CVal >= -4096 && CVal <= 4094) {
3699 int64_t Adj = CVal < 0 ? -2048 : 2047;
3700 Base = SDValue(
3701 CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3702 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3703 0);
3704 Offset = CurDAG->getSignedTargetConstant(Val: CVal - Adj, DL, VT);
3705 return true;
3706 }
3707
3708 // For larger immediates, we might be able to save one instruction from
3709 // constant materialization by folding the Lo12 bits of the immediate into
3710 // the address. We should only do this if the ADD is only used by loads and
3711 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
3712 // separately with the full materialized immediate creating extra
3713 // instructions.
3714 if (isWorthFoldingAdd(Add: Addr) &&
3715 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3716 Offset, /*IsPrefetch=*/false)) {
3717 // Insert an ADD instruction with the materialized Hi52 bits.
3718 Base = SDValue(
3719 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3720 0);
3721 return true;
3722 }
3723 }
3724
3725 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3726 /*IsPrefetch=*/false))
3727 return true;
3728
3729 Base = Addr;
3730 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3731 return true;
3732}
3733
3734/// Similar to SelectAddrRegImm, but only matches a register, or a register
3735/// plus a simm12 offset. Doesn't match a FrameIndex or global address, since
3736/// those aren't valid for the callers of this function (e.g. the target of
3737/// an indirect branch).
3738bool RISCVDAGToDAGISel::SelectBrindRegImm(SDValue Addr, SDValue &Base,
3739 SDValue &Offset) {
3740 SDLoc DL(Addr);
3741 MVT VT = Addr.getSimpleValueType();
3742
3743 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3744 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3745 if (isInt<12>(x: CVal)) {
3746 Base = Addr.getOperand(i: 0);
3747 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3748 return true;
3749 }
3750 }
3751
3752 Base = Addr;
3753 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3754 return true;
3755}
3756
3757/// Similar to SelectAddrRegImm, except that the offset is a 26-bit signed
3758/// immediate. This is used by the Qualcomm Xqcilo large offset load/store
3759/// instructions (qc.e.lw/qc.e.sw), whose offset field is 26 bits wide.
3760/// Only matches offsets that do not fit a 12-bit signed immediate, so that
3761/// offsets in the simm12 range keep using the shorter (and possibly
3762/// compressible) standard load/store instructions.
3763bool RISCVDAGToDAGISel::SelectAddrRegImm26(SDValue Addr, SDValue &Base,
3764 SDValue &Offset) {
3765 SDLoc DL(Addr);
3766 MVT VT = Addr.getSimpleValueType();
3767
3768 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3769 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3770 // Fold a 26-bit (but not 12-bit) signed offset directly into the
3771 // load/store.
3772 if (isInt<26>(x: CVal) && !isInt<12>(x: CVal)) {
3773 Base = Addr.getOperand(i: 0);
3774 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3775 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3776 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3777 return true;
3778 }
3779 }
3780
3781 // The offset is just outside the 26-bit range. Split off a small (simm12)
3782 // adjustment with a plain ADDI and fold the remaining 26-bit offset into the
3783 // load/store. A plain ADDI is used (rather than the wide
3784 // qc.e.addi/qc.e.addai) because the adjustment fits simm12: this keeps it a
3785 // short, compressible (c.addi) instruction and is available without Xqcilia.
3786 //
3787 // Skip the split if the address is used other than as a foldable load/store
3788 // base. `isWorthFoldingAdd()` returns true when every user of the add node is
3789 // a scalar load/store using it as an address operand. If it return false, it
3790 // means that some use consumes the add result as a value (e.g. it feeds
3791 // another add, is a stored value, is used in arithmetic) and that use forces
3792 // the add to be materialized into a register.
3793 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)) &&
3794 isWorthFoldingAdd(Add: Addr)) {
3795 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3796 if (!isInt<26>(x: CVal)) {
3797 // check if lw in lui + add + lw combination can be compressed.
3798 // The check here purely based on the immediate value and hopes that
3799 // register allocator would assign a register from a GPRC set so that the
3800 // instruction can get compressed.
3801 bool IsLwCompressable = isShiftedUInt<5, 2>(x: CVal & ((1 << 12) - 1));
3802
3803 int64_t Imm26 = CVal < 0 ? minIntN(N: 26) : maxIntN(N: 26);
3804 int64_t Adj = CVal - Imm26;
3805 // If Adj fits within 6-bits, then both combinations will take 8 bytes
3806 // however c.addi + qc.e.lw/sw will take 1 less cycle. Also, if lw is not
3807 // compressable then both combination would take 10 bytes but again
3808 // addi + qc.e.lw/sw will take 1 less cycle.
3809 if (isInt<6>(x: Adj) || (isInt<12>(x: Adj) && !IsLwCompressable)) {
3810 Base = SDValue(CurDAG->getMachineNode(
3811 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3812 Op2: CurDAG->getSignedTargetConstant(Val: Adj, DL, VT)),
3813 0);
3814 Offset = CurDAG->getSignedTargetConstant(Val: Imm26, DL, VT);
3815 return true;
3816 }
3817 }
3818 }
3819
3820 // Don't match: let the standard addressing modes handle it.
3821 return false;
3822}
3823
3824/// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
3825bool RISCVDAGToDAGISel::SelectAddrRegImm9(SDValue Addr, SDValue &Base,
3826 SDValue &Offset) {
3827 if (SelectAddrFrameIndex(Addr, Base, Offset))
3828 return true;
3829
3830 SDLoc DL(Addr);
3831 MVT VT = Addr.getSimpleValueType();
3832
3833 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3834 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3835 if (isUInt<9>(x: CVal)) {
3836 Base = Addr.getOperand(i: 0);
3837
3838 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3839 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3840 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3841 return true;
3842 }
3843 }
3844
3845 Base = Addr;
3846 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3847 return true;
3848}
3849
3850/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
3851/// Offset should be all zeros.
3852bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
3853 SDValue &Offset) {
3854 if (SelectAddrFrameIndex(Addr, Base, Offset))
3855 return true;
3856
3857 SDLoc DL(Addr);
3858 MVT VT = Addr.getSimpleValueType();
3859
3860 if (CurDAG->isBaseWithConstantOffset(Op: Addr)) {
3861 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3862 if (isInt<12>(x: CVal)) {
3863 Base = Addr.getOperand(i: 0);
3864
3865 // Early-out if not a valid offset.
3866 if ((CVal & 0b11111) != 0) {
3867 Base = Addr;
3868 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3869 return true;
3870 }
3871
3872 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Val&: Base))
3873 Base = CurDAG->getTargetFrameIndex(FI: FIN->getIndex(), VT);
3874 Offset = CurDAG->getSignedTargetConstant(Val: CVal, DL, VT);
3875 return true;
3876 }
3877 }
3878
3879 // Handle ADD with large immediates.
3880 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Val: Addr.getOperand(i: 1))) {
3881 int64_t CVal = cast<ConstantSDNode>(Val: Addr.getOperand(i: 1))->getSExtValue();
3882 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3883
3884 // Handle immediates in the range [-4096,-2049] or [2017, 4063]. We can save
3885 // one instruction by folding adjustment (-2048 or 2016) into the address.
3886 // The upper bound keeps CVal - 2016 within simm12 ([−2048, 2047]).
3887 if ((-2049 >= CVal && CVal >= -4096) || (4063 >= CVal && CVal >= 2017)) {
3888 int64_t Adj = CVal < 0 ? -2048 : 2016;
3889 int64_t AdjustedOffset = CVal - Adj;
3890 Base =
3891 SDValue(CurDAG->getMachineNode(
3892 Opcode: RISCV::ADDI, dl: DL, VT, Op1: Addr.getOperand(i: 0),
3893 Op2: CurDAG->getSignedTargetConstant(Val: AdjustedOffset, DL, VT)),
3894 0);
3895 Offset = CurDAG->getSignedTargetConstant(Val: Adj, DL, VT);
3896 return true;
3897 }
3898
3899 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr: Addr.getOperand(i: 1), Base,
3900 Offset, /*IsPrefetch=*/true)) {
3901 // Insert an ADD instruction with the materialized Hi52 bits.
3902 Base = SDValue(
3903 CurDAG->getMachineNode(Opcode: RISCV::ADD, dl: DL, VT, Op1: Addr.getOperand(i: 0), Op2: Base),
3904 0);
3905 return true;
3906 }
3907 }
3908
3909 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3910 /*IsPrefetch=*/true))
3911 return true;
3912
3913 Base = Addr;
3914 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT);
3915 return true;
3916}
3917
3918/// Return true if this a load/store that we have a RegRegScale instruction for.
3919static bool isRegRegScaleLoadOrStore(SDNode *User, SDValue Add,
3920 const RISCVSubtarget &Subtarget) {
3921 unsigned UserOpc = User->getOpcode();
3922 if (UserOpc != ISD::LOAD && UserOpc != ISD::STORE)
3923 return false;
3924 EVT VT = cast<MemSDNode>(Val: User)->getMemoryVT();
3925 // Zilx only provides indexed loads, so it must not enable reg+reg-scale
3926 // address folding for stores. XTheadMemIdx and Xqcisls have scaled stores.
3927 bool HasScalarIntegerMemIdx =
3928 Subtarget.hasVendorXTHeadMemIdx() || Subtarget.hasVendorXqcisls() ||
3929 (Subtarget.hasStdExtZilx() && UserOpc == ISD::LOAD);
3930 if (!(VT.isScalarInteger() && HasScalarIntegerMemIdx) &&
3931 !((VT == MVT::f32 || VT == MVT::f64) &&
3932 Subtarget.hasVendorXTHeadFMemIdx()))
3933 return false;
3934 // Don't allow stores of the value. It must be used as the address.
3935 if (UserOpc == ISD::STORE && cast<StoreSDNode>(Val: User)->getValue() == Add)
3936 return false;
3937
3938 return true;
3939}
3940
3941/// Is it profitable to fold this Add into RegRegScale load/store. If \p
3942/// Shift is non-null, then we have matched a shl+add. We allow reassociating
3943/// (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2)) if there is a
3944/// single addi and we don't have a SHXADD instruction we could use.
3945/// FIXME: May still need to check how many and what kind of users the SHL has.
3946static bool isWorthFoldingIntoRegRegScale(const RISCVSubtarget &Subtarget,
3947 SDValue Add,
3948 SDValue Shift = SDValue()) {
3949 bool FoundADDI = false;
3950 for (auto *User : Add->users()) {
3951 if (isRegRegScaleLoadOrStore(User, Add, Subtarget))
3952 continue;
3953
3954 // Allow a single ADDI that is used by loads/stores if we matched a shift.
3955 if (!Shift || FoundADDI || User->getOpcode() != ISD::ADD ||
3956 !isa<ConstantSDNode>(Val: User->getOperand(Num: 1)) ||
3957 !isInt<12>(x: cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue()))
3958 return false;
3959
3960 FoundADDI = true;
3961
3962 // If we have a SHXADD instruction, prefer that over reassociating an ADDI.
3963 assert(Shift.getOpcode() == ISD::SHL);
3964 unsigned ShiftAmt = Shift.getConstantOperandVal(i: 1);
3965 if (Subtarget.hasShlAdd(ShAmt: ShiftAmt))
3966 return false;
3967
3968 // All users of the ADDI should be load/store.
3969 for (auto *ADDIUser : User->users())
3970 if (!isRegRegScaleLoadOrStore(User: ADDIUser, Add: SDValue(User, 0), Subtarget))
3971 return false;
3972 }
3973
3974 return true;
3975}
3976
3977bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
3978 ArrayRef<unsigned> Amounts,
3979 SDValue &Base, SDValue &Index,
3980 SDValue &Scale) {
3981 if (Addr.getOpcode() != ISD::ADD)
3982 return false;
3983 SDValue LHS = Addr.getOperand(i: 0);
3984 SDValue RHS = Addr.getOperand(i: 1);
3985
3986 EVT VT = Addr.getSimpleValueType();
3987 auto SelectShl = [this, VT, Amounts](SDValue N, SDValue &Index,
3988 SDValue &Shift) {
3989 if (N.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
3990 return false;
3991
3992 // Only match shifts by a value in range [0, MaxShiftAmount].
3993 unsigned ShiftAmt = N.getConstantOperandVal(i: 1);
3994 if (!llvm::is_contained(Range: Amounts, Element: ShiftAmt))
3995 return false;
3996
3997 Index = N.getOperand(i: 0);
3998 Shift = CurDAG->getTargetConstant(Val: ShiftAmt, DL: SDLoc(N), VT);
3999 return true;
4000 };
4001
4002 if (auto *C1 = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4003 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
4004 if (LHS.getOpcode() == ISD::ADD &&
4005 !isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
4006 isInt<12>(x: C1->getSExtValue())) {
4007 if (SelectShl(LHS.getOperand(i: 1), Index, Scale) &&
4008 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 1))) {
4009 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
4010 DL: SDLoc(Addr), VT);
4011 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
4012 Op1: LHS.getOperand(i: 0), Op2: C1Val),
4013 0);
4014 return true;
4015 }
4016
4017 // Add is commutative so we need to check both operands.
4018 if (SelectShl(LHS.getOperand(i: 0), Index, Scale) &&
4019 isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: LHS, Shift: LHS.getOperand(i: 0))) {
4020 SDValue C1Val = CurDAG->getTargetConstant(Val: *C1->getConstantIntValue(),
4021 DL: SDLoc(Addr), VT);
4022 Base = SDValue(CurDAG->getMachineNode(Opcode: RISCV::ADDI, dl: SDLoc(Addr), VT,
4023 Op1: LHS.getOperand(i: 1), Op2: C1Val),
4024 0);
4025 return true;
4026 }
4027 }
4028
4029 // Don't match add with constants.
4030 // FIXME: Is this profitable for large constants that have 0s in the lower
4031 // 12 bits that we can materialize with LUI?
4032 return false;
4033 }
4034
4035 // Try to match a shift on the RHS.
4036 if (SelectShl(RHS, Index, Scale)) {
4037 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: RHS))
4038 return false;
4039 Base = LHS;
4040 return true;
4041 }
4042
4043 // Try to match a shift on the LHS.
4044 if (SelectShl(LHS, Index, Scale)) {
4045 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr, Shift: LHS))
4046 return false;
4047 Base = RHS;
4048 return true;
4049 }
4050
4051 if (!isWorthFoldingIntoRegRegScale(Subtarget: *Subtarget, Add: Addr))
4052 return false;
4053
4054 // Bail out if 0 is not in candidate shift amounts.
4055 if (!llvm::is_contained(Range&: Amounts, Element: 0))
4056 return false;
4057
4058 Base = LHS;
4059 Index = RHS;
4060 Scale = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(Addr), VT);
4061 return true;
4062}
4063
4064bool RISCVDAGToDAGISel::SelectAddrRegZextRegScale(SDValue Addr,
4065 ArrayRef<unsigned> Amounts,
4066 unsigned Bits, SDValue &Base,
4067 SDValue &Index,
4068 SDValue &Scale) {
4069 if (!SelectAddrRegRegScale(Addr, Amounts, Base, Index, Scale))
4070 return false;
4071
4072 if (Index.getOpcode() == ISD::AND) {
4073 auto *C = dyn_cast<ConstantSDNode>(Val: Index.getOperand(i: 1));
4074 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4075 Index = Index.getOperand(i: 0);
4076 return true;
4077 }
4078 }
4079
4080 return false;
4081}
4082
4083bool RISCVDAGToDAGISel::SelectAddrRegReg(SDValue Addr, SDValue &Base,
4084 SDValue &Offset) {
4085 if (Addr.getOpcode() != ISD::ADD)
4086 return false;
4087
4088 if (isa<ConstantSDNode>(Val: Addr.getOperand(i: 1)))
4089 return false;
4090
4091 Base = Addr.getOperand(i: 0);
4092 Offset = Addr.getOperand(i: 1);
4093 return true;
4094}
4095
4096bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
4097 SDValue &ShAmt) {
4098 ShAmt = N;
4099
4100 // Peek through zext.
4101 if (ShAmt->getOpcode() == ISD::ZERO_EXTEND)
4102 ShAmt = ShAmt.getOperand(i: 0);
4103
4104 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
4105 // amount. If there is an AND on the shift amount, we can bypass it if it
4106 // doesn't affect any of those bits.
4107 if (ShAmt.getOpcode() == ISD::AND &&
4108 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4109 const APInt &AndMask = ShAmt.getConstantOperandAPInt(i: 1);
4110
4111 // Since the max shift amount is a power of 2 we can subtract 1 to make a
4112 // mask that covers the bits needed to represent all shift amounts.
4113 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
4114 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
4115
4116 if (ShMask.isSubsetOf(RHS: AndMask)) {
4117 ShAmt = ShAmt.getOperand(i: 0);
4118 } else {
4119 // SimplifyDemandedBits may have optimized the mask so try restoring any
4120 // bits that are known zero.
4121 KnownBits Known = CurDAG->computeKnownBits(Op: ShAmt.getOperand(i: 0));
4122 if (!ShMask.isSubsetOf(RHS: AndMask | Known.Zero))
4123 return true;
4124 ShAmt = ShAmt.getOperand(i: 0);
4125 }
4126 }
4127
4128 if (ShAmt.getOpcode() == ISD::ADD &&
4129 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 1))) {
4130 uint64_t Imm = ShAmt.getConstantOperandVal(i: 1);
4131 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
4132 // to avoid the ADD.
4133 if (Imm != 0 && Imm % ShiftWidth == 0) {
4134 ShAmt = ShAmt.getOperand(i: 0);
4135 return true;
4136 }
4137 } else if (ShAmt.getOpcode() == ISD::SUB &&
4138 isa<ConstantSDNode>(Val: ShAmt.getOperand(i: 0))) {
4139 uint64_t Imm = ShAmt.getConstantOperandVal(i: 0);
4140 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
4141 // generate a NEG instead of a SUB of a constant.
4142 if (Imm != 0 && Imm % ShiftWidth == 0) {
4143 SDLoc DL(ShAmt);
4144 EVT VT = ShAmt.getValueType();
4145 SDValue Zero = CurDAG->getRegister(Reg: RISCV::X0, VT);
4146 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
4147 MachineSDNode *Neg = CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero,
4148 Op2: ShAmt.getOperand(i: 1));
4149 ShAmt = SDValue(Neg, 0);
4150 return true;
4151 }
4152 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4153 // to generate a NOT instead of a SUB of a constant.
4154 if (Imm % ShiftWidth == ShiftWidth - 1) {
4155 SDLoc DL(ShAmt);
4156 EVT VT = ShAmt.getValueType();
4157 MachineSDNode *Not = CurDAG->getMachineNode(
4158 Opcode: RISCV::XORI, dl: DL, VT, Op1: ShAmt.getOperand(i: 1),
4159 Op2: CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/IsTarget: true));
4160 ShAmt = SDValue(Not, 0);
4161 return true;
4162 }
4163 }
4164
4165 return true;
4166}
4167
4168/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
4169/// check for equality with 0. This function emits instructions that convert the
4170/// seteq/setne into something that can be compared with 0.
4171/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
4172/// ISD::SETNE).
4173bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
4174 SDValue &Val, bool OneUse) {
4175 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
4176 "Unexpected condition code!");
4177
4178 // We're looking for a setcc.
4179 if (N->getOpcode() != ISD::SETCC)
4180 return false;
4181
4182 if (OneUse && !N->hasOneUse())
4183 return false;
4184
4185 // Must be an equality comparison.
4186 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
4187 if (CCVal != ExpectedCCVal)
4188 return false;
4189
4190 SDValue LHS = N->getOperand(Num: 0);
4191 SDValue RHS = N->getOperand(Num: 1);
4192
4193 if (!LHS.getValueType().isScalarInteger())
4194 return false;
4195
4196 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
4197 if (isNullConstant(V: RHS)) {
4198 Val = LHS;
4199 return true;
4200 }
4201
4202 SDLoc DL(N);
4203
4204 if (auto *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
4205 int64_t CVal = C->getSExtValue();
4206 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
4207 // non-zero otherwise.
4208 if (CVal == -2048) {
4209 Val = SDValue(
4210 CurDAG->getMachineNode(
4211 Opcode: RISCV::XORI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4212 Op2: CurDAG->getSignedTargetConstant(Val: CVal, DL, VT: N->getValueType(ResNo: 0))),
4213 0);
4214 return true;
4215 }
4216 // If the RHS is [-2047,2048], we can use addi/addiw with -RHS to produce 0
4217 // if the LHS is equal to the RHS and non-zero otherwise.
4218 if (isInt<12>(x: CVal) || CVal == 2048) {
4219 unsigned Opc = RISCV::ADDI;
4220 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4221 cast<VTSDNode>(Val: LHS.getOperand(i: 1))->getVT() == MVT::i32) {
4222 Opc = RISCV::ADDIW;
4223 LHS = LHS.getOperand(i: 0);
4224 }
4225
4226 Val = SDValue(CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4227 Op2: CurDAG->getSignedTargetConstant(
4228 Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4229 0);
4230 return true;
4231 }
4232 if (isPowerOf2_64(Value: CVal) && Subtarget->hasStdExtZbs()) {
4233 Val = SDValue(
4234 CurDAG->getMachineNode(
4235 Opcode: RISCV::BINVI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4236 Op2: CurDAG->getTargetConstant(Val: Log2_64(Value: CVal), DL, VT: N->getValueType(ResNo: 0))),
4237 0);
4238 return true;
4239 }
4240 // Same as the addi case above but for larger immediates (signed 26-bit) use
4241 // the QC_E_ADDI instruction from the Xqcilia extension, if available. Avoid
4242 // anything which can be done with a single lui as it might be compressible.
4243 if (Subtarget->hasVendorXqcilia() && isInt<26>(x: CVal) &&
4244 (CVal & 0xFFF) != 0) {
4245 Val = SDValue(
4246 CurDAG->getMachineNode(
4247 Opcode: RISCV::QC_E_ADDI, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS,
4248 Op2: CurDAG->getSignedTargetConstant(Val: -CVal, DL, VT: N->getValueType(ResNo: 0))),
4249 0);
4250 return true;
4251 }
4252 }
4253
4254 // If nothing else we can XOR the LHS and RHS to produce zero if they are
4255 // equal and a non-zero value if they aren't.
4256 Val = SDValue(
4257 CurDAG->getMachineNode(Opcode: RISCV::XOR, dl: DL, VT: N->getValueType(ResNo: 0), Op1: LHS, Op2: RHS), 0);
4258 return true;
4259}
4260
4261bool RISCVDAGToDAGISel::selectSExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4262 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4263 cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT().getSizeInBits() == Bits) {
4264 Val = N.getOperand(i: 0);
4265 return true;
4266 }
4267
4268 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
4269 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(Val: N.getOperand(i: 1)))
4270 return N;
4271
4272 SDValue N0 = N.getOperand(i: 0);
4273 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4274 N.getConstantOperandVal(i: 1) == ShiftAmt &&
4275 N0.getConstantOperandVal(i: 1) == ShiftAmt)
4276 return N0.getOperand(i: 0);
4277
4278 return N;
4279 };
4280
4281 MVT VT = N.getSimpleValueType();
4282 if (CurDAG->ComputeNumSignBits(Op: N) > (VT.getSizeInBits() - Bits)) {
4283 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
4284 return true;
4285 }
4286
4287 return false;
4288}
4289
4290bool RISCVDAGToDAGISel::selectZExtBits(SDValue N, unsigned Bits, SDValue &Val) {
4291 if (N.getOpcode() == ISD::AND) {
4292 auto *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
4293 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(N: Bits)) {
4294 Val = N.getOperand(i: 0);
4295 return true;
4296 }
4297 }
4298 MVT VT = N.getSimpleValueType();
4299 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: Bits);
4300 if (CurDAG->MaskedValueIsZero(Op: N, Mask)) {
4301 Val = N;
4302 return true;
4303 }
4304
4305 return false;
4306}
4307
4308/// Look for various patterns that can be done with a SHL that can be folded
4309/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
4310/// SHXADD we are trying to match.
4311bool RISCVDAGToDAGISel::selectSHXADDOp(SDValue N, unsigned ShAmt,
4312 SDValue &Val) {
4313 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4314 MVT VT = N.getSimpleValueType();
4315 assert(VT == Subtarget->getXLenVT());
4316 SDValue N0 = N.getOperand(i: 0);
4317
4318 if (bool LeftShift = N0.getOpcode() == ISD::SHL;
4319 (LeftShift || N0.getOpcode() == ISD::SRL) &&
4320 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4321 uint64_t Mask = N.getConstantOperandVal(i: 1);
4322 unsigned C2 = N0.getConstantOperandVal(i: 1);
4323 assert(C2 > 0 && C2 < Subtarget->getXLen() && "Unexpected shift amount!");
4324
4325 unsigned XLen = Subtarget->getXLen();
4326 if (LeftShift)
4327 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4328 else
4329 Mask &= maskTrailingOnes<uint64_t>(N: XLen - C2);
4330
4331 if (isShiftedMask_64(Value: Mask)) {
4332 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4333 unsigned Trailing = llvm::countr_zero(Val: Mask);
4334 if (Trailing != ShAmt)
4335 return false;
4336
4337 unsigned Opcode;
4338 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
4339 // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
4340 // followed by a SHXADD with c3 for the X amount.
4341 if (LeftShift && Leading == 0 && C2 < Trailing)
4342 Opcode = RISCV::SRLI;
4343 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
4344 // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
4345 // followed by a SHXADD with c3 for the X amount.
4346 else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
4347 Opcode = RISCV::SRLIW;
4348 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
4349 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
4350 // followed by a SHXADD using c3 for the X amount.
4351 else if (!LeftShift && Leading == C2)
4352 Opcode = RISCV::SRLI;
4353 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
4354 // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
4355 // followed by a SHXADD using c3 for the X amount.
4356 else if (!LeftShift && Leading == 32 + C2)
4357 Opcode = RISCV::SRLIW;
4358 else
4359 return false;
4360
4361 SDLoc DL(N);
4362 ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
4363 Val = SDValue(
4364 CurDAG->getMachineNode(Opcode, dl: DL, VT, Op1: N0.getOperand(i: 0),
4365 Op2: CurDAG->getTargetConstant(Val: ShAmt, DL, VT)),
4366 0);
4367 return true;
4368 }
4369 } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
4370 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4371 uint64_t Mask = N.getConstantOperandVal(i: 1);
4372 unsigned C2 = N0.getConstantOperandVal(i: 1);
4373
4374 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
4375 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
4376 // use (srli (srai y, c2 - c3), c3 + c4) followed by a SHXADD with c4 as
4377 // the X amount.
4378 if (isShiftedMask_64(Value: Mask)) {
4379 unsigned XLen = Subtarget->getXLen();
4380 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4381 unsigned Trailing = llvm::countr_zero(Val: Mask);
4382 if (C2 > Leading && Leading > 0 && Trailing == ShAmt) {
4383 SDLoc DL(N);
4384 EVT VT = N.getValueType();
4385 Val = SDValue(CurDAG->getMachineNode(
4386 Opcode: RISCV::SRAI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4387 Op2: CurDAG->getTargetConstant(Val: C2 - Leading, DL, VT)),
4388 0);
4389 Val = SDValue(CurDAG->getMachineNode(
4390 Opcode: RISCV::SRLI, dl: DL, VT, Op1: Val,
4391 Op2: CurDAG->getTargetConstant(Val: Leading + ShAmt, DL, VT)),
4392 0);
4393 return true;
4394 }
4395 }
4396 }
4397 } else if (bool LeftShift = N.getOpcode() == ISD::SHL;
4398 (LeftShift || N.getOpcode() == ISD::SRL) &&
4399 isa<ConstantSDNode>(Val: N.getOperand(i: 1))) {
4400 MVT VT = N.getSimpleValueType();
4401 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
4402 SDValue N0 = N.getOperand(i: 0);
4403 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
4404 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
4405 uint64_t Mask = N0.getConstantOperandVal(i: 1);
4406 if (isShiftedMask_64(Value: Mask)) {
4407 unsigned C1 = N.getConstantOperandVal(i: 1);
4408 assert(C1 > 0 && C1 < Subtarget->getXLen() &&
4409 "Unexpected shift amount!");
4410 unsigned XLen = Subtarget->getXLen();
4411 unsigned Leading = XLen - llvm::bit_width(Value: Mask);
4412 unsigned Trailing = llvm::countr_zero(Val: Mask);
4413 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
4414 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
4415 if (LeftShift && Leading == 32 && Trailing > 0 &&
4416 (Trailing + C1) == ShAmt) {
4417 SDLoc DL(N);
4418 EVT VT = N.getValueType();
4419 Val = SDValue(CurDAG->getMachineNode(
4420 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4421 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4422 0);
4423 return true;
4424 }
4425 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
4426 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
4427 if (!LeftShift && Leading == 32 && Trailing > C1 &&
4428 (Trailing - C1) == ShAmt) {
4429 SDLoc DL(N);
4430 Val = SDValue(CurDAG->getMachineNode(
4431 Opcode: RISCV::SRLIW, dl: DL, VT, Op1: N0.getOperand(i: 0),
4432 Op2: CurDAG->getTargetConstant(Val: Trailing, DL, VT)),
4433 0);
4434 return true;
4435 }
4436 }
4437 }
4438 }
4439
4440 return false;
4441}
4442
4443/// Look for various patterns that can be done with a SHL that can be folded
4444/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
4445/// SHXADD_UW we are trying to match.
4446bool RISCVDAGToDAGISel::selectSHXADD_UWOp(SDValue N, unsigned ShAmt,
4447 SDValue &Val) {
4448 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
4449 N.hasOneUse()) {
4450 assert(N.getSimpleValueType() == Subtarget->getXLenVT() &&
4451 "Unexpected VT!");
4452 SDValue N0 = N.getOperand(i: 0);
4453 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
4454 N0.hasOneUse()) {
4455 uint64_t Mask = N.getConstantOperandVal(i: 1);
4456 unsigned C2 = N0.getConstantOperandVal(i: 1);
4457 assert(C2 > 0 && C2 < Subtarget->getXLen() && "Unexpected shift amount!");
4458
4459 Mask &= maskTrailingZeros<uint64_t>(N: C2);
4460
4461 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
4462 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
4463 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
4464 if (isShiftedMask_64(Value: Mask)) {
4465 unsigned Leading = llvm::countl_zero(Val: Mask);
4466 unsigned Trailing = llvm::countr_zero(Val: Mask);
4467 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
4468 SDLoc DL(N);
4469 EVT VT = N.getValueType();
4470 Val = SDValue(CurDAG->getMachineNode(
4471 Opcode: RISCV::SLLI, dl: DL, VT, Op1: N0.getOperand(i: 0),
4472 Op2: CurDAG->getTargetConstant(Val: C2 - ShAmt, DL, VT)),
4473 0);
4474 return true;
4475 }
4476 }
4477 }
4478 }
4479
4480 return false;
4481}
4482
4483bool RISCVDAGToDAGISel::orDisjoint(const SDNode *N) const {
4484 assert(N->getOpcode() == ISD::OR || N->getOpcode() == RISCVISD::OR_VL);
4485 if (N->getFlags().hasDisjoint())
4486 return true;
4487 return CurDAG->haveNoCommonBitsSet(A: N->getOperand(Num: 0), B: N->getOperand(Num: 1));
4488}
4489
4490bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
4491 SDValue N, SDValue &Val) {
4492 int OrigCost = RISCVMatInt::getIntMatCost(Val: APInt(64, OrigImm), Size: 64, STI: *Subtarget,
4493 /*CompressionCost=*/true);
4494 int Cost = RISCVMatInt::getIntMatCost(Val: APInt(64, Imm), Size: 64, STI: *Subtarget,
4495 /*CompressionCost=*/true);
4496 if (OrigCost <= Cost)
4497 return false;
4498
4499 Val = selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm, Subtarget: *Subtarget);
4500 return true;
4501}
4502
4503bool RISCVDAGToDAGISel::selectZExtImm32(SDValue N, SDValue &Val) {
4504 if (!isa<ConstantSDNode>(Val: N))
4505 return false;
4506 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4507 if ((Imm >> 31) != 1)
4508 return false;
4509
4510 for (const SDNode *U : N->users()) {
4511 switch (U->getOpcode()) {
4512 case ISD::ADD:
4513 break;
4514 case ISD::OR:
4515 if (orDisjoint(N: U))
4516 break;
4517 return false;
4518 default:
4519 return false;
4520 }
4521 }
4522
4523 return selectImm64IfCheaper(Imm: 0xffffffff00000000 | Imm, OrigImm: Imm, N, Val);
4524}
4525
4526bool RISCVDAGToDAGISel::selectNegImm(SDValue N, SDValue &Val) {
4527 if (!isa<ConstantSDNode>(Val: N))
4528 return false;
4529 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4530 if (isInt<32>(x: Imm))
4531 return false;
4532 if (Imm == INT64_MIN)
4533 return false;
4534
4535 for (const SDNode *U : N->users()) {
4536 switch (U->getOpcode()) {
4537 case ISD::ADD:
4538 break;
4539 case RISCVISD::VMV_V_X_VL:
4540 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4541 return V->getOpcode() == ISD::ADD ||
4542 V->getOpcode() == RISCVISD::ADD_VL;
4543 }))
4544 return false;
4545 break;
4546 default:
4547 return false;
4548 }
4549 }
4550
4551 return selectImm64IfCheaper(Imm: -Imm, OrigImm: Imm, N, Val);
4552}
4553
4554bool RISCVDAGToDAGISel::selectInvLogicImm(SDValue N, SDValue &Val) {
4555 if (!isa<ConstantSDNode>(Val: N))
4556 return false;
4557 int64_t Imm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
4558
4559 // For 32-bit signed constants, we can only substitute LUI+ADDI with LUI.
4560 if (isInt<32>(x: Imm) && ((Imm & 0xfff) != 0xfff || Imm == -1))
4561 return false;
4562
4563 // Abandon this transform if the constant is needed elsewhere.
4564 for (const SDNode *U : N->users()) {
4565 switch (U->getOpcode()) {
4566 case ISD::AND:
4567 case ISD::OR:
4568 case ISD::XOR:
4569 if (!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()))
4570 return false;
4571 break;
4572 case RISCVISD::VMV_V_X_VL:
4573 if (!Subtarget->hasStdExtZvkb())
4574 return false;
4575 if (!all_of(Range: U->users(), P: [](const SDNode *V) {
4576 return V->getOpcode() == ISD::AND ||
4577 V->getOpcode() == RISCVISD::AND_VL;
4578 }))
4579 return false;
4580 break;
4581 default:
4582 return false;
4583 }
4584 }
4585
4586 if (isInt<32>(x: Imm)) {
4587 Val =
4588 selectImm(CurDAG, DL: SDLoc(N), VT: N->getSimpleValueType(ResNo: 0), Imm: ~Imm, Subtarget: *Subtarget);
4589 return true;
4590 }
4591
4592 // For 64-bit constants, the instruction sequences get complex,
4593 // so we select inverted only if it's cheaper.
4594 return selectImm64IfCheaper(Imm: ~Imm, OrigImm: Imm, N, Val);
4595}
4596
4597static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
4598 unsigned Bits,
4599 const TargetInstrInfo *TII) {
4600 unsigned MCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: User->getMachineOpcode());
4601
4602 if (!MCOpcode)
4603 return false;
4604
4605 const MCInstrDesc &MCID = TII->get(Opcode: User->getMachineOpcode());
4606 const uint64_t TSFlags = MCID.TSFlags;
4607 if (!RISCVII::hasSEWOp(TSFlags))
4608 return false;
4609 assert(RISCVII::hasVLOp(TSFlags));
4610
4611 unsigned ChainOpIdx = User->getNumOperands() - 1;
4612 bool HasChainOp = User->getOperand(Num: ChainOpIdx).getValueType() == MVT::Other;
4613 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
4614 unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
4615 const unsigned Log2SEW = User->getConstantOperandVal(Num: VLIdx + 1);
4616
4617 if (UserOpNo == VLIdx)
4618 return false;
4619
4620 auto NumDemandedBits =
4621 RISCV::getVectorLowDemandedScalarBits(Opcode: MCOpcode, Log2SEW);
4622 return NumDemandedBits && Bits >= *NumDemandedBits;
4623}
4624
4625// Return true if all users of this SDNode* only consume the lower \p Bits.
4626// This can be used to form W instructions for add/sub/mul/shl even when the
4627// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
4628// SimplifyDemandedBits has made it so some users see a sext_inreg and some
4629// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
4630// the add/sub/mul/shl to become non-W instructions. By checking the users we
4631// may be able to use a W instruction and CSE with the other instruction if
4632// this has happened. We could try to detect that the CSE opportunity exists
4633// before doing this, but that would be more complicated.
4634bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits,
4635 const unsigned Depth) const {
4636 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
4637 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
4638 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
4639 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
4640 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
4641 isa<ConstantSDNode>(Node) || Depth != 0) &&
4642 "Unexpected opcode");
4643
4644 if (Depth >= SelectionDAG::MaxRecursionDepth)
4645 return false;
4646
4647 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
4648 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
4649 if (Depth == 0 && !Node->getValueType(ResNo: 0).isScalarInteger())
4650 return false;
4651
4652 for (SDUse &Use : Node->uses()) {
4653 SDNode *User = Use.getUser();
4654 // Users of this node should have already been instruction selected
4655 if (!User->isMachineOpcode())
4656 return false;
4657
4658 // TODO: Add more opcodes?
4659 switch (User->getMachineOpcode()) {
4660 default:
4661 if (vectorPseudoHasAllNBitUsers(User, UserOpNo: Use.getOperandNo(), Bits, TII))
4662 break;
4663 return false;
4664 case RISCV::ADDW:
4665 case RISCV::ADDIW:
4666 case RISCV::SUBW:
4667 case RISCV::MULW:
4668 case RISCV::SLLW:
4669 case RISCV::SLLIW:
4670 case RISCV::SRAW:
4671 case RISCV::SRAIW:
4672 case RISCV::SRLW:
4673 case RISCV::SRLIW:
4674 case RISCV::DIVW:
4675 case RISCV::DIVUW:
4676 case RISCV::REMW:
4677 case RISCV::REMUW:
4678 case RISCV::ROLW:
4679 case RISCV::RORW:
4680 case RISCV::RORIW:
4681 case RISCV::CLSW:
4682 case RISCV::CLZW:
4683 case RISCV::CTZW:
4684 case RISCV::CPOPW:
4685 case RISCV::SLLI_UW:
4686 case RISCV::ABSW:
4687 case RISCV::FMV_W_X:
4688 case RISCV::FCVT_H_W:
4689 case RISCV::FCVT_H_W_INX:
4690 case RISCV::FCVT_H_WU:
4691 case RISCV::FCVT_H_WU_INX:
4692 case RISCV::FCVT_S_W:
4693 case RISCV::FCVT_S_W_INX:
4694 case RISCV::FCVT_S_WU:
4695 case RISCV::FCVT_S_WU_INX:
4696 case RISCV::FCVT_D_W:
4697 case RISCV::FCVT_D_W_INX:
4698 case RISCV::FCVT_D_WU:
4699 case RISCV::FCVT_D_WU_INX:
4700 case RISCV::TH_REVW:
4701 case RISCV::TH_SRRIW:
4702 if (Bits >= 32)
4703 break;
4704 return false;
4705 case RISCV::SLL:
4706 case RISCV::SRA:
4707 case RISCV::SRL:
4708 case RISCV::ROL:
4709 case RISCV::ROR:
4710 case RISCV::BSET:
4711 case RISCV::BCLR:
4712 case RISCV::BINV:
4713 case RISCV::BEXT:
4714 // Shift amount and bit index operands only use log2(Xlen) bits.
4715 if (Use.getOperandNo() == 1 && Bits >= Log2_32(Value: Subtarget->getXLen()))
4716 break;
4717 return false;
4718 case RISCV::SLLI:
4719 // SLLI only uses the lower (XLen - ShAmt) bits.
4720 if (Bits >= Subtarget->getXLen() - User->getConstantOperandVal(Num: 1))
4721 break;
4722 return false;
4723 case RISCV::ANDI:
4724 if (Bits >= (unsigned)llvm::bit_width(Value: User->getConstantOperandVal(Num: 1)))
4725 break;
4726 goto RecCheck;
4727 case RISCV::ORI: {
4728 uint64_t Imm = cast<ConstantSDNode>(Val: User->getOperand(Num: 1))->getSExtValue();
4729 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(Value: ~Imm))
4730 break;
4731 [[fallthrough]];
4732 }
4733 case RISCV::AND:
4734 case RISCV::OR:
4735 case RISCV::XOR:
4736 case RISCV::XORI:
4737 case RISCV::ANDN:
4738 case RISCV::ORN:
4739 case RISCV::XNOR:
4740 case RISCV::SH1ADD:
4741 case RISCV::SH2ADD:
4742 case RISCV::SH3ADD:
4743 RecCheck:
4744 if (hasAllNBitUsers(Node: User, Bits, Depth: Depth + 1))
4745 break;
4746 return false;
4747 case RISCV::SRLI: {
4748 unsigned ShAmt = User->getConstantOperandVal(Num: 1);
4749 // If we are shifting right by less than Bits, and users don't demand any
4750 // bits that were shifted into [Bits-1:0], then we can consider this as an
4751 // N-Bit user.
4752 if (Bits > ShAmt && hasAllNBitUsers(Node: User, Bits: Bits - ShAmt, Depth: Depth + 1))
4753 break;
4754 return false;
4755 }
4756 case RISCV::SEXT_B:
4757 case RISCV::PACKH:
4758 if (Bits >= 8)
4759 break;
4760 return false;
4761 case RISCV::SEXT_H:
4762 case RISCV::FMV_H_X:
4763 case RISCV::ZEXT_H_RV32:
4764 case RISCV::ZEXT_H_RV64:
4765 case RISCV::PACKW:
4766 if (Bits >= 16)
4767 break;
4768 return false;
4769 case RISCV::PACK:
4770 if (Bits >= (Subtarget->getXLen() / 2))
4771 break;
4772 return false;
4773 case RISCV::PPAIRE_H:
4774 // If only the lower 32-bits of the result are used, then only the
4775 // lower 16 bits of the inputs are used.
4776 if (Bits >= 16 && hasAllNBitUsers(Node: User, Bits: 32, Depth: Depth + 1))
4777 break;
4778 return false;
4779 case RISCV::ADD_UW:
4780 case RISCV::SH1ADD_UW:
4781 case RISCV::SH2ADD_UW:
4782 case RISCV::SH3ADD_UW:
4783 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
4784 // 32 bits.
4785 if (Use.getOperandNo() == 0 && Bits >= 32)
4786 break;
4787 return false;
4788 case RISCV::SB:
4789 if (Use.getOperandNo() == 0 && Bits >= 8)
4790 break;
4791 return false;
4792 case RISCV::SH:
4793 if (Use.getOperandNo() == 0 && Bits >= 16)
4794 break;
4795 return false;
4796 case RISCV::SW:
4797 if (Use.getOperandNo() == 0 && Bits >= 32)
4798 break;
4799 return false;
4800 case RISCV::TH_EXT:
4801 case RISCV::TH_EXTU: {
4802 unsigned Msb = User->getConstantOperandVal(Num: 1);
4803 unsigned Lsb = User->getConstantOperandVal(Num: 2);
4804 // Behavior of Msb < Lsb is not well documented.
4805 if (Msb >= Lsb && Bits > Msb)
4806 break;
4807 return false;
4808 }
4809 }
4810 }
4811
4812 return true;
4813}
4814
4815// Select a constant that can be represented as (sign_extend(imm5) << imm2).
4816bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
4817 SDValue &Shl2) {
4818 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4819 if (!C)
4820 return false;
4821
4822 int64_t Offset = C->getSExtValue();
4823 for (unsigned Shift = 0; Shift < 4; Shift++) {
4824 if (isInt<5>(x: Offset >> Shift) && ((Offset % (1LL << Shift)) == 0)) {
4825 EVT VT = N->getValueType(ResNo: 0);
4826 Simm5 = CurDAG->getSignedTargetConstant(Val: Offset >> Shift, DL: SDLoc(N), VT);
4827 Shl2 = CurDAG->getTargetConstant(Val: Shift, DL: SDLoc(N), VT);
4828 return true;
4829 }
4830 }
4831
4832 return false;
4833}
4834
4835// Select VL as a 5 bit immediate or a value that will become a register. This
4836// allows us to choose between VSETIVLI or VSETVLI later.
4837bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
4838 auto *C = dyn_cast<ConstantSDNode>(Val&: N);
4839 if (C && isUInt<5>(x: C->getZExtValue())) {
4840 VL = CurDAG->getTargetConstant(Val: C->getZExtValue(), DL: SDLoc(N),
4841 VT: N->getValueType(ResNo: 0));
4842 } else if (C && C->isAllOnes()) {
4843 // Treat all ones as VLMax.
4844 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4845 VT: N->getValueType(ResNo: 0));
4846 } else if (isa<RegisterSDNode>(Val: N) &&
4847 cast<RegisterSDNode>(Val&: N)->getReg() == RISCV::X0) {
4848 // All our VL operands use an operand that allows GPRNoX0 or an immediate
4849 // as the register class. Convert X0 to a special immediate to pass the
4850 // MachineVerifier. This is recognized specially by the vsetvli insertion
4851 // pass.
4852 VL = CurDAG->getSignedTargetConstant(Val: RISCV::VLMaxSentinel, DL: SDLoc(N),
4853 VT: N->getValueType(ResNo: 0));
4854 } else {
4855 VL = N;
4856 }
4857
4858 return true;
4859}
4860
4861static SDValue findVSplat(SDValue N) {
4862 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
4863 if (!N.getOperand(i: 0).isUndef())
4864 return SDValue();
4865 N = N.getOperand(i: 1);
4866 }
4867 SDValue Splat = N;
4868 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
4869 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
4870 !Splat.getOperand(i: 0).isUndef())
4871 return SDValue();
4872 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
4873 return Splat;
4874}
4875
4876bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
4877 SDValue Splat = findVSplat(N);
4878 if (!Splat)
4879 return false;
4880
4881 SplatVal = Splat.getOperand(i: 1);
4882 return true;
4883}
4884
4885static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
4886 SelectionDAG &DAG,
4887 const RISCVSubtarget &Subtarget,
4888 std::function<bool(int64_t)> ValidateImm,
4889 bool Decrement = false) {
4890 SDValue Splat = findVSplat(N);
4891 if (!Splat || !isa<ConstantSDNode>(Val: Splat.getOperand(i: 1)))
4892 return false;
4893
4894 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
4895 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
4896 "Unexpected splat operand type");
4897
4898 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
4899 // type is wider than the resulting vector element type: an implicit
4900 // truncation first takes place. Therefore, perform a manual
4901 // truncation/sign-extension in order to ignore any truncated bits and catch
4902 // any zero-extended immediate.
4903 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
4904 // sign-extending to (XLenVT -1).
4905 APInt SplatConst = Splat.getConstantOperandAPInt(i: 1).sextOrTrunc(width: SplatEltSize);
4906
4907 int64_t SplatImm = SplatConst.getSExtValue();
4908
4909 if (!ValidateImm(SplatImm))
4910 return false;
4911
4912 if (Decrement)
4913 SplatImm -= 1;
4914
4915 SplatVal =
4916 DAG.getSignedTargetConstant(Val: SplatImm, DL: SDLoc(N), VT: Subtarget.getXLenVT());
4917 return true;
4918}
4919
4920bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
4921 return selectVSplatImmHelper(N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4922 ValidateImm: [](int64_t Imm) { return isInt<5>(x: Imm); });
4923}
4924
4925bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
4926 return selectVSplatImmHelper(
4927 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4928 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4929 /*Decrement=*/true);
4930}
4931
4932bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal) {
4933 return selectVSplatImmHelper(
4934 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4935 ValidateImm: [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4936 /*Decrement=*/false);
4937}
4938
4939bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
4940 SDValue &SplatVal) {
4941 return selectVSplatImmHelper(
4942 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4943 ValidateImm: [](int64_t Imm) { return Imm != 0 && Imm >= -15 && Imm <= 16; },
4944 /*Decrement=*/true);
4945}
4946
4947bool RISCVDAGToDAGISel::selectVSplatUimm(SDValue N, unsigned Bits,
4948 SDValue &SplatVal) {
4949 return selectVSplatImmHelper(
4950 N, SplatVal, DAG&: *CurDAG, Subtarget: *Subtarget,
4951 ValidateImm: [Bits](int64_t Imm) { return isUIntN(N: Bits, x: Imm); });
4952}
4953
4954bool RISCVDAGToDAGISel::selectVSplatImm64Neg(SDValue N, SDValue &SplatVal) {
4955 SDValue Splat = findVSplat(N);
4956 return Splat && selectNegImm(N: Splat.getOperand(i: 1), Val&: SplatVal);
4957}
4958
4959bool RISCVDAGToDAGISel::selectLow8BitsVSplat(SDValue N, SDValue &SplatVal) {
4960 auto IsExtOrTrunc = [](SDValue N) {
4961 switch (N->getOpcode()) {
4962 case ISD::SIGN_EXTEND:
4963 case ISD::ZERO_EXTEND:
4964 // There's no passthru on these _VL nodes so any VL/mask is ok, since any
4965 // inactive elements will be undef.
4966 case RISCVISD::TRUNCATE_VECTOR_VL:
4967 case RISCVISD::VSEXT_VL:
4968 case RISCVISD::VZEXT_VL:
4969 return true;
4970 default:
4971 return false;
4972 }
4973 };
4974
4975 // We can have multiple nested nodes, so unravel them all if needed.
4976 while (IsExtOrTrunc(N)) {
4977 if (!N.hasOneUse() || N.getScalarValueSizeInBits() < 8)
4978 return false;
4979 N = N->getOperand(Num: 0);
4980 }
4981
4982 return selectVSplat(N, SplatVal);
4983}
4984
4985bool RISCVDAGToDAGISel::selectScalarFPAsInt(SDValue N, SDValue &Imm) {
4986 // Allow bitcasts from XLenVT -> FP.
4987 if (N.getOpcode() == ISD::BITCAST &&
4988 N.getOperand(i: 0).getValueType() == Subtarget->getXLenVT()) {
4989 Imm = N.getOperand(i: 0);
4990 return true;
4991 }
4992 // Allow moves from XLenVT to FP.
4993 if (N.getOpcode() == RISCVISD::FMV_H_X ||
4994 N.getOpcode() == RISCVISD::FMV_W_X_RV64) {
4995 Imm = N.getOperand(i: 0);
4996 return true;
4997 }
4998
4999 // Otherwise, look for FP constants that can materialized with scalar int.
5000 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val: N.getNode());
5001 if (!CFP)
5002 return false;
5003 const APFloat &APF = CFP->getValueAPF();
5004 // td can handle +0.0 already.
5005 if (APF.isPosZero())
5006 return false;
5007
5008 MVT VT = CFP->getSimpleValueType(ResNo: 0);
5009
5010 MVT XLenVT = Subtarget->getXLenVT();
5011 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
5012 assert(APF.isNegZero() && "Unexpected constant.");
5013 return false;
5014 }
5015 SDLoc DL(N);
5016 Imm = selectImm(CurDAG, DL, VT: XLenVT, Imm: APF.bitcastToAPInt().getSExtValue(),
5017 Subtarget: *Subtarget);
5018 return true;
5019}
5020
5021bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
5022 SDValue &Imm) {
5023 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N)) {
5024 int64_t ImmVal = SignExtend64(X: C->getSExtValue(), B: Width);
5025
5026 if (!isInt<5>(x: ImmVal))
5027 return false;
5028
5029 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N),
5030 VT: Subtarget->getXLenVT());
5031 return true;
5032 }
5033
5034 return false;
5035}
5036
5037// Match XOR with a VMSET_VL operand. Return the other operand.
5038bool RISCVDAGToDAGISel::selectVMNOTOp(SDValue N, SDValue &Res) {
5039 if (N.getOpcode() != ISD::XOR)
5040 return false;
5041
5042 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5043 Res = N.getOperand(i: 1);
5044 return true;
5045 }
5046
5047 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5048 Res = N.getOperand(i: 0);
5049 return true;
5050 }
5051
5052 return false;
5053}
5054
5055// Match VMXOR_VL with a VMSET_VL operand. Making sure that that VL operand
5056// matches the parent's VL. Return the other operand of the VMXOR_VL.
5057bool RISCVDAGToDAGISel::selectVMNOT_VLOp(SDNode *Parent, SDValue N,
5058 SDValue &Res) {
5059 if (N.getOpcode() != RISCVISD::VMXOR_VL)
5060 return false;
5061
5062 assert(Parent &&
5063 (Parent->getOpcode() == RISCVISD::VMAND_VL ||
5064 Parent->getOpcode() == RISCVISD::VMOR_VL ||
5065 Parent->getOpcode() == RISCVISD::VMXOR_VL) &&
5066 "Unexpected parent");
5067
5068 // The VL should match the parent.
5069 if (Parent->getOperand(Num: 2) != N->getOperand(Num: 2))
5070 return false;
5071
5072 if (N.getOperand(i: 0).getOpcode() == RISCVISD::VMSET_VL) {
5073 Res = N.getOperand(i: 1);
5074 return true;
5075 }
5076
5077 if (N.getOperand(i: 1).getOpcode() == RISCVISD::VMSET_VL) {
5078 Res = N.getOperand(i: 0);
5079 return true;
5080 }
5081
5082 return false;
5083}
5084
5085// Try to remove sext.w if the input is a W instruction or can be made into
5086// a W instruction cheaply.
5087bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
5088 // Look for the sext.w pattern, addiw rd, rs1, 0.
5089 if (N->getMachineOpcode() != RISCV::ADDIW ||
5090 !isNullConstant(V: N->getOperand(Num: 1)))
5091 return false;
5092
5093 SDValue N0 = N->getOperand(Num: 0);
5094 if (!N0.isMachineOpcode())
5095 return false;
5096
5097 switch (N0.getMachineOpcode()) {
5098 default:
5099 break;
5100 case RISCV::ADD:
5101 case RISCV::ADDI:
5102 case RISCV::SUB:
5103 case RISCV::MUL:
5104 case RISCV::SLLI: {
5105 // Convert sext.w+add/sub/mul to their W instructions. This will create
5106 // a new independent instruction. This improves latency.
5107 unsigned Opc;
5108 switch (N0.getMachineOpcode()) {
5109 default:
5110 llvm_unreachable("Unexpected opcode!");
5111 case RISCV::ADD: Opc = RISCV::ADDW; break;
5112 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
5113 case RISCV::SUB: Opc = RISCV::SUBW; break;
5114 case RISCV::MUL: Opc = RISCV::MULW; break;
5115 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
5116 }
5117
5118 SDValue N00 = N0.getOperand(i: 0);
5119 SDValue N01 = N0.getOperand(i: 1);
5120
5121 // Shift amount needs to be uimm5.
5122 if (N0.getMachineOpcode() == RISCV::SLLI &&
5123 !isUInt<5>(x: cast<ConstantSDNode>(Val&: N01)->getSExtValue()))
5124 break;
5125
5126 SDNode *Result =
5127 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VT: N->getValueType(ResNo: 0),
5128 Op1: N00, Op2: N01);
5129 ReplaceUses(F: N, T: Result);
5130 return true;
5131 }
5132 case RISCV::ADDW:
5133 case RISCV::ADDIW:
5134 case RISCV::SUBW:
5135 case RISCV::MULW:
5136 case RISCV::SLLIW:
5137 case RISCV::PACKW:
5138 case RISCV::TH_MULAW:
5139 case RISCV::TH_MULAH:
5140 case RISCV::TH_MULSW:
5141 case RISCV::TH_MULSH:
5142 if (N0.getValueType() == MVT::i32)
5143 break;
5144
5145 // Result is already sign extended just remove the sext.w.
5146 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
5147 ReplaceUses(F: N, T: N0.getNode());
5148 return true;
5149 }
5150
5151 return false;
5152}
5153
5154static bool usesAllOnesMask(SDValue MaskOp) {
5155 const auto IsVMSet = [](unsigned Opc) {
5156 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
5157 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
5158 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
5159 Opc == RISCV::PseudoVMSET_M_B8;
5160 };
5161
5162 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
5163 // undefined behaviour if it's the wrong bitwidth, so we could choose to
5164 // assume that it's all-ones? Same applies to its VL.
5165 return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode());
5166}
5167
5168static bool isImplicitDef(SDValue V) {
5169 if (!V.isMachineOpcode())
5170 return false;
5171 if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
5172 for (unsigned I = 1; I < V.getNumOperands(); I += 2)
5173 if (!isImplicitDef(V: V.getOperand(i: I)))
5174 return false;
5175 return true;
5176 }
5177 return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
5178}
5179
5180// Optimize masked RVV pseudo instructions with a known all-ones mask to their
5181// corresponding "unmasked" pseudo versions.
5182bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
5183 const RISCV::RISCVMaskedPseudoInfo *I =
5184 RISCV::getMaskedPseudoInfo(MaskedPseudo: N->getMachineOpcode());
5185 if (!I)
5186 return false;
5187
5188 unsigned MaskOpIdx = I->MaskOpIdx;
5189 if (!usesAllOnesMask(MaskOp: N->getOperand(Num: MaskOpIdx)))
5190 return false;
5191
5192 // There are two classes of pseudos in the table - compares and
5193 // everything else. See the comment on RISCVMaskedPseudo for details.
5194 const unsigned Opc = I->UnmaskedPseudo;
5195 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
5196 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MCID);
5197
5198 const MCInstrDesc &MaskedMCID = TII->get(Opcode: N->getMachineOpcode());
5199 const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MaskedMCID);
5200
5201 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
5202 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&
5203 "Unmasked pseudo has policy but masked pseudo doesn't?");
5204 assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru &&
5205 "Unexpected pseudo structure");
5206 assert(!(HasPassthru && !MaskedHasPassthru) &&
5207 "Unmasked pseudo has passthru but masked pseudo doesn't?");
5208
5209 SmallVector<SDValue, 8> Ops;
5210 // Skip the passthru operand at index 0 if the unmasked don't have one.
5211 bool ShouldSkip = !HasPassthru && MaskedHasPassthru;
5212 bool DropPolicy = !RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags) &&
5213 RISCVII::hasVecPolicyOp(TSFlags: MaskedMCID.TSFlags);
5214 bool HasChainOp =
5215 N->getOperand(Num: N->getNumOperands() - 1).getValueType() == MVT::Other;
5216 unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp;
5217 for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) {
5218 // Skip the mask
5219 SDValue Op = N->getOperand(Num: I);
5220 if (I == MaskOpIdx)
5221 continue;
5222 if (DropPolicy && I == LastOpNum)
5223 continue;
5224 Ops.push_back(Elt: Op);
5225 }
5226
5227 MachineSDNode *Result =
5228 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5229
5230 if (!N->memoperands_empty())
5231 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: N->memoperands());
5232
5233 Result->setFlags(N->getFlags());
5234 ReplaceUses(F: N, T: Result);
5235
5236 return true;
5237}
5238
5239/// If our passthru is an implicit_def, use noreg instead. This side
5240/// steps issues with MachineCSE not being able to CSE expressions with
5241/// IMPLICIT_DEF operands while preserving the semantic intent. See
5242/// pr64282 for context. Note that this transform is the last one
5243/// performed at ISEL DAG to DAG.
5244bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
5245 bool MadeChange = false;
5246 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5247
5248 while (Position != CurDAG->allnodes_begin()) {
5249 SDNode *N = &*--Position;
5250 if (N->use_empty() || !N->isMachineOpcode())
5251 continue;
5252
5253 const unsigned Opc = N->getMachineOpcode();
5254 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opc) ||
5255 !RISCVII::isFirstDefTiedToFirstUse(Desc: TII->get(Opcode: Opc)) ||
5256 !isImplicitDef(V: N->getOperand(Num: 0)))
5257 continue;
5258
5259 SmallVector<SDValue> Ops;
5260 Ops.push_back(Elt: CurDAG->getRegister(Reg: RISCV::NoRegister, VT: N->getValueType(ResNo: 0)));
5261 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
5262 SDValue Op = N->getOperand(Num: I);
5263 Ops.push_back(Elt: Op);
5264 }
5265
5266 MachineSDNode *Result =
5267 CurDAG->getMachineNode(Opcode: Opc, dl: SDLoc(N), VTs: N->getVTList(), Ops);
5268 Result->setFlags(N->getFlags());
5269 CurDAG->setNodeMemRefs(N: Result, NewMemRefs: cast<MachineSDNode>(Val: N)->memoperands());
5270 ReplaceUses(F: N, T: Result);
5271 MadeChange = true;
5272 }
5273 return MadeChange;
5274}
5275
5276
5277// This pass converts a legalized DAG into a RISCV-specific DAG, ready
5278// for instruction scheduling.
5279FunctionPass *llvm::createRISCVISelDagLegacyPass(RISCVTargetMachine &TM,
5280 CodeGenOptLevel OptLevel) {
5281 return new RISCVDAGToDAGISelLegacy(TM, OptLevel);
5282}
5283
5284RISCVISelDAGToDAGPass::RISCVISelDAGToDAGPass(RISCVTargetMachine &TM,
5285 CodeGenOptLevel OptLevel)
5286 : SelectionDAGISelPass(std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5287
5288char RISCVDAGToDAGISelLegacy::ID = 0;
5289
5290RISCVDAGToDAGISelLegacy::RISCVDAGToDAGISelLegacy(RISCVTargetMachine &TM,
5291 CodeGenOptLevel OptLevel)
5292 : SelectionDAGISelLegacy(
5293 ID, std::make_unique<RISCVDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
5294
5295INITIALIZE_PASS(RISCVDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
5296