1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "RISCVSelectionDAGInfo.h"
10#include "RISCVSubtarget.h"
11#include "llvm/CodeGen/SelectionDAG.h"
12
13#define GET_SDNODE_DESC
14#include "RISCVGenSDNodeInfo.inc"
15
16using namespace llvm;
17
18RISCVSelectionDAGInfo::RISCVSelectionDAGInfo()
19 : SelectionDAGGenTargetInfo(RISCVGenSDNodeInfo) {}
20
21RISCVSelectionDAGInfo::~RISCVSelectionDAGInfo() = default;
22
23void RISCVSelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG,
24 const SDNode *N) const {
25 SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N);
26
27#ifndef NDEBUG
28 // Some additional checks not yet implemented by verifyTargetNode.
29 switch (N->getOpcode()) {
30 case RISCVISD::TUPLE_EXTRACT:
31 assert(N->getOperand(1).getOpcode() == ISD::TargetConstant &&
32 "Expected index to be a target constant!");
33 break;
34 case RISCVISD::TUPLE_INSERT:
35 assert(N->getOperand(2).getOpcode() == ISD::TargetConstant &&
36 "Expected index to be a target constant!");
37 break;
38 case RISCVISD::TUPLE_CAST: {
39 EVT VT = N->getValueType(0);
40 EVT OpVT = N->getOperand(0).getValueType();
41 assert(VT.isRISCVVectorTuple() && OpVT.isRISCVVectorTuple() &&
42 "Expected input and output of TUPLE_CAST to be vector tuples");
43 unsigned NF = VT.getRISCVVectorTupleNumFields();
44 unsigned OpNF = OpVT.getRISCVVectorTupleNumFields();
45 unsigned LMUL = divideCeil(VT.getSizeInBits().getKnownMinValue(),
46 NF * RISCV::RVVBitsPerBlock);
47 unsigned OpLMUL = divideCeil(OpVT.getSizeInBits().getKnownMinValue(),
48 OpNF * RISCV::RVVBitsPerBlock);
49 assert(NF == OpNF && LMUL == OpLMUL &&
50 "Expected input and output of TUPLE_CAST to have the same "
51 "factor and LMUL");
52 break;
53 }
54 case RISCVISD::VDOT4A_VL:
55 case RISCVISD::VDOT4AU_VL:
56 case RISCVISD::VDOT4ASU_VL: {
57 EVT VT = N->getValueType(0);
58 assert(VT.isScalableVectorOf(MVT::i32) &&
59 "Expected result to be an i32 scalable vector");
60 assert(N->getOperand(0).getValueType() == VT &&
61 N->getOperand(1).getValueType() == VT &&
62 N->getOperand(2).getValueType() == VT &&
63 "Expected result and first 3 operands to have the same type!");
64 EVT MaskVT = N->getOperand(3).getValueType();
65 assert(MaskVT.isScalableVector() &&
66 MaskVT.getVectorElementCount() == VT.getVectorElementCount() &&
67 "Expected mask VT to be an i1 scalable vector with same number of "
68 "elements as the result");
69 break;
70 }
71 case RISCVISD::PNCLIPP:
72 case RISCVISD::PNCLIPUP: {
73 EVT VT = N->getValueType(0);
74 EVT OpVT = N->getOperand(0).getValueType();
75 assert(VT.isVector() && "Expected vector result");
76 assert(VT.getSizeInBits() == OpVT.getSizeInBits() &&
77 "Expected result and operands to have the same size!");
78 assert(N->getOperand(1).getValueType() == OpVT &&
79 "Expected operands to have the same type");
80 assert(((!OpVT.isVector() && VT.getVectorNumElements() == 2) ||
81 (OpVT.isVector() &&
82 OpVT.getVectorNumElements() * 2 == VT.getVectorNumElements())) &&
83 "Expected operands to be scalar or a vector with half the number of "
84 "elements");
85 break;
86 }
87 }
88#endif
89}
90
91SDValue RISCVSelectionDAGInfo::EmitTargetCodeForMemset(
92 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
93 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
94 MachinePointerInfo DstPtrInfo) const {
95 const auto &Subtarget = DAG.getSubtarget<RISCVSubtarget>();
96 // We currently do this only for Xqcilsm
97 if (!Subtarget.hasVendorXqcilsm())
98 return SDValue();
99
100 // Do this only if we know the size at compile time.
101 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
102 if (!ConstantSize)
103 return SDValue();
104
105 uint64_t NumberOfBytesToWrite = ConstantSize->getZExtValue();
106
107 // Do this only if it is word aligned and we write a multiple of 4 bytes.
108 if (!(Alignment >= 4) || !((NumberOfBytesToWrite & 3) == 0))
109 return SDValue();
110
111 SmallVector<SDValue, 8> OutChains;
112 SDValue SrcValueReplicated = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i32, Operand: Src);
113 int NumberOfWords = NumberOfBytesToWrite / 4;
114 MachineFunction &MF = DAG.getMachineFunction();
115 auto Volatile =
116 isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
117
118 // Helper for constructing the QC_SETWMI instruction
119 auto getSetwmiNode = [&](uint8_t SizeWords, uint8_t OffsetSetwmi) -> SDValue {
120 SDValue Ops[] = {Chain, SrcValueReplicated, Dst,
121 DAG.getTargetConstant(Val: SizeWords, DL: dl, VT: MVT::i32),
122 DAG.getTargetConstant(Val: OffsetSetwmi, DL: dl, VT: MVT::i32)};
123 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
124 PtrInfo: DstPtrInfo.getWithOffset(O: OffsetSetwmi),
125 F: MachineMemOperand::MOStore | Volatile, Size: SizeWords * 4, BaseAlignment: Align(4));
126 return DAG.getMemIntrinsicNode(Opcode: RISCVISD::QC_SETWMI, dl,
127 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT: MVT::i32,
128 MMO: BaseMemOperand);
129 };
130
131 // If i8 type and constant non-zero value.
132 if ((Src.getValueType() == MVT::i8) && !isNullConstant(V: Src))
133 // Replicate byte to word by multiplication with 0x01010101.
134 SrcValueReplicated =
135 DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32, N1: SrcValueReplicated,
136 N2: DAG.getConstant(Val: 0x01010101ul, DL: dl, VT: MVT::i32));
137
138 // We limit a QC_SETWMI to 16 words or less to improve interruptibility.
139 // So for 1-16 words we use a single QC_SETWMI:
140 //
141 // QC_SETWMI reg1, N, 0(reg2)
142 //
143 // For 17-32 words we use two QC_SETWMI's with the first as 16 words and the
144 // second for the remainder:
145 //
146 // QC_SETWMI reg1, 16, 0(reg2)
147 // QC_SETWMI reg1, N, 64(reg2)
148 //
149 // For 33-48 words, we would like to use (16, 16, n), but that means the last
150 // QC_SETWMI needs an offset of 128 which the instruction doesn't support.
151 // So in this case we use a length of 15 for the second instruction and we do
152 // the rest with the third instruction.
153 // This means the maximum inlined number of words is 47 (for now):
154 //
155 // QC_SETWMI R2, R0, 16, 0
156 // QC_SETWMI R2, R0, 15, 64
157 // QC_SETWMI R2, R0, N, 124
158 //
159 // For 48 words or more, call the target independent memset
160 if (NumberOfWords >= 48)
161 return SDValue();
162
163 if (NumberOfWords <= 16) {
164 // 1 - 16 words
165 return getSetwmiNode(NumberOfWords, 0);
166 }
167
168 if (NumberOfWords <= 32) {
169 // 17 - 32 words
170 OutChains.push_back(Elt: getSetwmiNode(NumberOfWords - 16, 64));
171 OutChains.push_back(Elt: getSetwmiNode(16, 0));
172 } else {
173 // 33 - 47 words
174 OutChains.push_back(Elt: getSetwmiNode(NumberOfWords - 31, 124));
175 OutChains.push_back(Elt: getSetwmiNode(15, 64));
176 OutChains.push_back(Elt: getSetwmiNode(16, 0));
177 }
178
179 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
180}
181