1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for RISC-V.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "RISCVLegalizerInfo.h"
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVSubtarget.h"
17#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/MachineConstantPool.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/TargetOpcodes.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsRISCV.h"
32#include "llvm/IR/Type.h"
33
34using namespace llvm;
35using namespace LegalityPredicates;
36using namespace LegalizeMutations;
37using namespace MIPatternMatch;
38
39static LegalityPredicate
40typeIsLegalIntOrFPVec(unsigned TypeIdx,
41 std::initializer_list<LLT> IntOrFPVecTys,
42 const RISCVSubtarget &ST) {
43 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
44 return ST.hasVInstructions() &&
45 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
46 ST.hasVInstructionsI64()) &&
47 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
48 ST.getELen() == 64);
49 };
50
51 return all(P0: typeInSet(TypeIdx, TypesInit: IntOrFPVecTys), P1: P);
52}
53
54static LegalityPredicate
55typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
56 const RISCVSubtarget &ST) {
57 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
58 return ST.hasVInstructions() &&
59 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
60 ST.getELen() == 64);
61 };
62 return all(P0: typeInSet(TypeIdx, TypesInit: BoolVecTys), P1: P);
63}
64
65static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
66 std::initializer_list<LLT> PtrVecTys,
67 const RISCVSubtarget &ST) {
68 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
69 return ST.hasVInstructions() &&
70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
71 ST.getELen() == 64) &&
72 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
73 Query.Types[TypeIdx].getScalarSizeInBits() == 32);
74 };
75 return all(P0: typeInSet(TypeIdx, TypesInit: PtrVecTys), P1: P);
76}
77
78RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
79 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(SizeInBits: XLen)),
80 iXLen(LLT::integer(SizeInBits: XLen)) {
81 const LLT sDoubleXLen = LLT::scalar(SizeInBits: 2 * XLen);
82 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: XLen);
83 const LLT s1 = LLT::scalar(SizeInBits: 1);
84 const LLT s8 = LLT::scalar(SizeInBits: 8);
85 const LLT s16 = LLT::scalar(SizeInBits: 16);
86 const LLT f16 = LLT::float16();
87 const LLT f32 = LLT::float32();
88 const LLT f64 = LLT::float64();
89 const LLT f128 = LLT::float128();
90 const LLT s32 = LLT::scalar(SizeInBits: 32);
91 const LLT s64 = LLT::scalar(SizeInBits: 64);
92 const LLT s128 = LLT::scalar(SizeInBits: 128);
93
94 const LLT nxv1s1 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s1);
95 const LLT nxv2s1 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s1);
96 const LLT nxv4s1 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s1);
97 const LLT nxv8s1 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s1);
98 const LLT nxv16s1 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s1);
99 const LLT nxv32s1 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s1);
100 const LLT nxv64s1 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s1);
101
102 const LLT nxv1s8 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s8);
103 const LLT nxv2s8 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s8);
104 const LLT nxv4s8 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s8);
105 const LLT nxv8s8 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s8);
106 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
107 const LLT nxv32s8 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s8);
108 const LLT nxv64s8 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s8);
109
110 const LLT nxv1s16 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s16);
111 const LLT nxv2s16 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s16);
112 const LLT nxv4s16 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s16);
113 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
114 const LLT nxv16s16 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s16);
115 const LLT nxv32s16 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s16);
116
117 const LLT nxv1s32 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s32);
118 const LLT nxv2s32 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s32);
119 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
120 const LLT nxv8s32 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s32);
121 const LLT nxv16s32 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s32);
122
123 const LLT nxv1s64 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s64);
124 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
125 const LLT nxv4s64 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s64);
126 const LLT nxv8s64 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s64);
127
128 const LLT nxv1p0 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: p0);
129 const LLT nxv2p0 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: p0);
130 const LLT nxv4p0 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: p0);
131 const LLT nxv8p0 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: p0);
132 const LLT nxv16p0 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: p0);
133
134 using namespace TargetOpcode;
135
136 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
137
138 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,
139 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
140 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
141 nxv1s64, nxv2s64, nxv4s64, nxv8s64};
142
143 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
144
145 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
146 .legalFor(Types: {sXLen})
147 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
148 .customFor(Pred: ST.is64Bit(), Types: {s32})
149 .widenScalarToNextPow2(TypeIdx: 0)
150 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
151
152 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
153 .legalFor(Types: {sXLen})
154 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
155 .widenScalarToNextPow2(TypeIdx: 0)
156 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
157
158 getActionDefinitionsBuilder(
159 Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO, G_READ_REGISTER, G_WRITE_REGISTER})
160 .lower();
161
162 getActionDefinitionsBuilder(Opcodes: {G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})
163 .minScalar(TypeIdx: 0, Ty: sXLen)
164 .lower();
165
166 // TODO: Use Vector Single-Width Saturating Instructions for vector types.
167 getActionDefinitionsBuilder(
168 Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})
169 .lower();
170
171 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
172 .legalFor(Types: {{sXLen, sXLen}})
173 .customFor(Pred: ST.is64Bit(), Types: {{s32, s32}})
174 .widenScalarToNextPow2(TypeIdx: 0)
175 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
176 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
177
178 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
179 .legalFor(Types: {{s32, s16}})
180 .legalFor(Pred: ST.is64Bit(), Types: {{s64, s16}, {s64, s32}})
181 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
182 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
183 .customIf(Predicate: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))
184 .maxScalar(TypeIdx: 0, Ty: sXLen);
185
186 getActionDefinitionsBuilder(Opcode: G_TRUNC).alwaysLegal();
187
188 {
189 LegalityPredicate ValidSextInRegWidth = all(P0: sizeIs(TypeIdx: 0, Size: 64), P1: immIs(ImmIdx: 0, Imm: 32));
190
191 if (STI.hasStdExtZbb())
192 ValidSextInRegWidth =
193 LegalityPredicates::any(P0: ValidSextInRegWidth, P1: immInSet(ImmIdx: 0, ImmsInit: {8, 16}));
194
195 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
196 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: sXLen), P1: ValidSextInRegWidth))
197 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
198 .lower();
199 }
200
201 // Merge/Unmerge
202 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
203 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Opcode: Op);
204 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
205 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
206 if (XLen == 32 && (ST.hasStdExtD() || ST.hasStdExtZilsd())) {
207 MergeUnmergeActions.legalIf(
208 Predicate: all(P0: typeIs(TypeIdx: BigTyIdx, TypesInit: s64), P1: typeIs(TypeIdx: LitTyIdx, TypesInit: s32)));
209 }
210 MergeUnmergeActions.widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: XLen)
211 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: XLen)
212 .clampScalar(TypeIdx: LitTyIdx, MinTy: sXLen, MaxTy: sXLen)
213 .clampScalar(TypeIdx: BigTyIdx, MinTy: sXLen, MaxTy: sXLen);
214 }
215
216 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR}).lower();
217
218 getActionDefinitionsBuilder(Opcodes: {G_ROTR, G_ROTL})
219 .legalFor(Pred: ST.hasStdExtZbb() || ST.hasStdExtZbkb(), Types: {{sXLen, sXLen}})
220 .customFor(Pred: ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),
221 Types: {{s32, s32}})
222 .lower();
223
224 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
225 .customFor(Pred: ST.hasStdExtZbkb(), Types: {s8})
226 .lowerIf(Predicate: scalarNarrowerThan(TypeIdx: 0, Size: 8))
227 .widenScalarToNextPow2(TypeIdx: 0)
228 .maxScalar(TypeIdx: 0, Ty: sXLen)
229 .lower();
230
231 getActionDefinitionsBuilder(Opcode: G_BITCAST)
232 .legalIf(Predicate: all(
233 P0: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
234 P1: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST)),
235 P1: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST),
236 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))))
237 // Scalar int <-> fp bitcasts that fit in a GPR are register moves.
238 .legalIf(Predicate: all(P0: isScalar(TypeIdx: 0), P1: isScalar(TypeIdx: 1), args: scalarNarrowerThan(TypeIdx: 0, Size: XLen + 1)))
239 // Wider ones (i128/f128, or f64/i64 on RV32) are split into XLen pieces.
240 .narrowScalarIf(Predicate: all(P0: isScalar(TypeIdx: 0), P1: isScalar(TypeIdx: 1), args: scalarWiderThan(TypeIdx: 0, Size: XLen)),
241 Mutation: LegalizeMutations::changeTo(TypeIdx: 0, Ty: sXLen));
242
243 auto &BSWAPActions = getActionDefinitionsBuilder(Opcode: G_BSWAP);
244 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
245 BSWAPActions.legalFor(Types: {sXLen}).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
246 else
247 BSWAPActions.maxScalar(TypeIdx: 0, Ty: sXLen).lower();
248
249 getActionDefinitionsBuilder(Opcode: G_CLMUL)
250 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
251 .unsupported();
252
253 getActionDefinitionsBuilder(Opcode: G_CLMULH)
254 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
255 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbkc(), Types: {s32})
256 .unsupported();
257
258 // CLMULR is Zbc-only; Zbkc is a subset that has CLMUL/CLMULH but not CLMULR.
259 getActionDefinitionsBuilder(Opcode: G_CLMULR)
260 .legalFor(Pred: ST.hasStdExtZbc(), Types: {sXLen})
261 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbc(), Types: {s32})
262 .unsupported();
263
264 auto &CTLZPoisonActions = getActionDefinitionsBuilder(Opcode: G_CTLZ_ZERO_POISON);
265 auto &CTTZPoisonActions = getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON);
266 if (ST.hasStdExtZbb()) {
267 getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTTZ})
268 .legalFor(Types: {{sXLen, sXLen}})
269 .customFor(Types: {{s32, s32}})
270 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
271 .widenScalarToNextPow2(TypeIdx: 0)
272 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
273 } else {
274 auto &CTLZActions = getActionDefinitionsBuilder(Opcode: G_CTLZ);
275 if (ST.hasVendorXTHeadBb())
276 CTLZActions.legalFor(Types: {{sXLen, sXLen}})
277 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
278 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
279 else
280 CTLZActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
281
282 auto &CTTZActions = getActionDefinitionsBuilder(Opcode: G_CTTZ);
283 if (ST.hasVendorXTHeadBb()) {
284 CTTZActions.clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
285 CTLZPoisonActions.clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
286 CTTZPoisonActions.clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
287 } else {
288 CTTZActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
289 CTLZPoisonActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
290 CTTZPoisonActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
291 }
292 }
293 CTLZPoisonActions.lower();
294 CTTZPoisonActions.lower();
295
296 auto &CountSignActions = getActionDefinitionsBuilder(Opcode: G_CTLS);
297 if (ST.hasStdExtP()) {
298 CountSignActions.legalFor(Types: {{sXLen, sXLen}})
299 .customFor(Types: {{s32, s32}})
300 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
301 .widenScalarToNextPow2(TypeIdx: 0)
302 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
303 } else {
304 CountSignActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
305 }
306
307 auto &CTPOPActions = getActionDefinitionsBuilder(Opcode: G_CTPOP);
308 if (ST.hasStdExtZbb()) {
309 CTPOPActions.legalFor(Types: {{sXLen, sXLen}})
310 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
311 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
312 } else {
313 CTPOPActions.widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8)
314 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sXLen)
315 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0)
316 .lower();
317 }
318
319 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
320 .legalFor(Types: {p0})
321 .legalFor(Pred: !ST.is64Bit(), Types: {s32})
322 .customFor(Pred: ST.is64Bit(), Types: {s64})
323 .widenScalarToNextPow2(TypeIdx: 0)
324 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
325
326 // TODO: transform illegal vector types into legal vector type
327 getActionDefinitionsBuilder(Opcode: G_FREEZE)
328 .legalFor(Types: {s16, s32, p0})
329 .legalFor(Pred: ST.is64Bit(), Types: {s64})
330 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
331 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
332 .widenScalarToNextPow2(TypeIdx: 0)
333 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen);
334
335 // TODO: transform illegal vector types into legal vector type
336 // TODO: Merge with G_FREEZE?
337 getActionDefinitionsBuilder(
338 Opcodes: {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})
339 .legalFor(Types: {s32, sXLen, p0})
340 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
341 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
342 .widenScalarToNextPow2(TypeIdx: 0)
343 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
344
345 getActionDefinitionsBuilder(Opcode: G_ICMP)
346 .legalFor(Types: {{sXLen, sXLen}, {sXLen, p0}})
347 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
348 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
349 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, MinSize: 8)
350 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
351 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
352
353 getActionDefinitionsBuilder(Opcode: G_SELECT)
354 .legalFor(Types: {{s32, sXLen}, {p0, sXLen}})
355 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
356 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
357 .legalFor(Pred: XLen == 64 || ST.hasStdExtD(), Types: {{s64, sXLen}})
358 .widenScalarToNextPow2(TypeIdx: 0)
359 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)
360 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
361
362 auto &LoadActions = getActionDefinitionsBuilder(Opcode: G_LOAD);
363 auto &StoreActions = getActionDefinitionsBuilder(Opcode: G_STORE);
364 auto &ExtLoadActions = getActionDefinitionsBuilder(Opcodes: {G_SEXTLOAD, G_ZEXTLOAD});
365
366 // Return the alignment needed for scalar memory ops. If unaligned scalar mem
367 // is supported, we only require byte alignment. Otherwise, we need the memory
368 // op to be natively aligned.
369 auto getScalarMemAlign = [&ST](unsigned Size) {
370 return ST.enableUnalignedScalarMem() ? 8 : Size;
371 };
372
373 LoadActions.legalForTypesWithMemDesc(
374 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
375 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
376 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
377 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
378 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
379 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
380 StoreActions.legalForTypesWithMemDesc(
381 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
382 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
383 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
384 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
385 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
386 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
387 ExtLoadActions.legalForTypesWithMemDesc(
388 TypesAndMemDesc: {{.Type0: sXLen, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
389 {.Type0: sXLen, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)}});
390 if (XLen == 64) {
391 LoadActions.legalForTypesWithMemDesc(
392 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
393 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
394 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
395 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
396 StoreActions.legalForTypesWithMemDesc(
397 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
398 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
399 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
400 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
401 ExtLoadActions.legalForTypesWithMemDesc(
402 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)}});
403 } else if (ST.hasStdExtD() || ST.hasStdExtZilsd()) {
404 // RV32: s64 is legal for f64 (D/Zdinx) or i64 (Zilsd); LLT cannot tell the
405 // two apart, so one rule covers both.
406 // FIXME: We require natural alignment, so under -mzilsd-word-align an
407 // align-4 i64 access is split into two s32 ones. riscv-prera-zilsd-opt
408 // merges them back into LD_RV32/SD_RV32, but it is skipped at -O0.
409 LoadActions.legalForTypesWithMemDesc(
410 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
411 StoreActions.legalForTypesWithMemDesc(
412 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
413 }
414
415 // Vector loads/stores.
416 if (ST.hasVInstructions()) {
417 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
418 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
419 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
420 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
421 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
422 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
423 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
424 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
425 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
426 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
427 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
428 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
429 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
430 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
431 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
432 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
433 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
434 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
435 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
436 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
437 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
438 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
439 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
440 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
441 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
442 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
443 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
444 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
445 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
446 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
447
448 if (ST.getELen() == 64) {
449 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
450 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
451 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
452 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
453 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
454 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
455 }
456
457 if (ST.hasVInstructionsI64()) {
458 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
459 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
460 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
461 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
462 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
463 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
464 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
465 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
466 }
467
468 // we will take the custom lowering logic if we have scalable vector types
469 // with non-standard alignments
470 LoadActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
471 StoreActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
472
473 // Pointers require that XLen sized elements are legal.
474 if (XLen <= ST.getELen()) {
475 LoadActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
476 StoreActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
477 }
478 }
479
480 LoadActions.widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
481 .lowerIfMemSizeNotByteSizePow2()
482 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
483 .lower();
484 StoreActions
485 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
486 .lowerIfMemSizeNotByteSizePow2()
487 .lower();
488
489 ExtLoadActions.widenScalarToNextPow2(TypeIdx: 0).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).lower();
490
491 getActionDefinitionsBuilder(Opcodes: {G_PTR_ADD, G_PTRMASK}).legalFor(Types: {{p0, sXLen}});
492
493 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
494 .legalFor(Types: {{sXLen, p0}})
495 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
496
497 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
498 .legalFor(Types: {{p0, sXLen}})
499 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
500
501 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
502
503 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {sXLen}).minScalar(TypeIdx: 0, Ty: sXLen);
504
505 getActionDefinitionsBuilder(Opcode: G_BRJT).customFor(Types: {{p0, sXLen}});
506
507 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
508
509 getActionDefinitionsBuilder(Opcode: G_PHI)
510 .legalFor(Types: {p0, s32, sXLen})
511 .widenScalarToNextPow2(TypeIdx: 0)
512 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
513
514 getActionDefinitionsBuilder(Opcodes: {G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
515 .legalFor(Types: {p0});
516
517 if (ST.hasStdExtZmmul()) {
518 getActionDefinitionsBuilder(Opcode: G_MUL)
519 .legalFor(Types: {sXLen})
520 .widenScalarToNextPow2(TypeIdx: 0)
521 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
522
523 // clang-format off
524 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
525 .legalFor(Types: {sXLen})
526 .lower();
527 // clang-format on
528
529 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO}).minScalar(TypeIdx: 0, Ty: sXLen).lower();
530 } else {
531 getActionDefinitionsBuilder(Opcode: G_MUL)
532 .libcallFor(Types: {sXLen, sDoubleXLen})
533 .widenScalarToNextPow2(TypeIdx: 0)
534 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen);
535
536 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).lowerFor(Types: {sXLen});
537
538 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
539 .minScalar(TypeIdx: 0, Ty: sXLen)
540 // Widen sXLen to sDoubleXLen so we can use a single libcall to get
541 // the low bits for the mul result and high bits to do the overflow
542 // check.
543 .widenScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: sXLen),
544 Mutation: LegalizeMutations::changeTo(TypeIdx: 0, Ty: sDoubleXLen))
545 .lower();
546 }
547
548 if (ST.hasStdExtM()) {
549 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV, G_UREM})
550 .legalFor(Types: {sXLen})
551 .customFor(Types: {s32})
552 .libcallFor(Types: {sDoubleXLen})
553 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sDoubleXLen)
554 .widenScalarToNextPow2(TypeIdx: 0);
555 getActionDefinitionsBuilder(Opcode: G_SREM)
556 .legalFor(Types: {sXLen})
557 .libcallFor(Types: {sDoubleXLen})
558 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
559 .widenScalarToNextPow2(TypeIdx: 0);
560 } else {
561 getActionDefinitionsBuilder(Opcodes: {G_UDIV, G_SDIV, G_UREM, G_SREM})
562 .libcallFor(Types: {sXLen, sDoubleXLen})
563 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
564 .widenScalarToNextPow2(TypeIdx: 0);
565 }
566
567 // TODO: Use libcall for sDoubleXLen.
568 getActionDefinitionsBuilder(Opcodes: {G_SDIVREM, G_UDIVREM}).lower();
569
570 getActionDefinitionsBuilder(Opcode: G_ABS)
571 .customFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
572 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
573 .lower();
574
575 getActionDefinitionsBuilder(Opcodes: {G_ABDS, G_ABDU})
576 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
577 .lower();
578
579 getActionDefinitionsBuilder(Opcodes: {G_UMAX, G_UMIN, G_SMAX, G_SMIN})
580 .legalFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
581 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
582 .lower();
583
584 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
585
586 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
587
588 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
589
590 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
591
592 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
593 .lower();
594
595 // On RV64 the 64-bit counter CSRs (cycle/time) are read directly. On RV32
596 // they are custom-legally lowered to a re-read-the-high-half loop (see
597 // legalizeReadCounter).
598 getActionDefinitionsBuilder(Opcodes: {G_READCYCLECOUNTER, G_READSTEADYCOUNTER})
599 .legalFor(Pred: ST.is64Bit(), Types: {s64})
600 .customFor(Pred: !ST.is64Bit(), Types: {s64});
601
602 // FP Operations
603
604 // FIXME: Support f128 for rv32 when libcall handling is able to use sret.
605 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,
606 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,
607 G_FMINIMUMNUM})
608 .legalFor(Pred: ST.hasStdExtF(), Types: {f32})
609 .legalFor(Pred: ST.hasStdExtD(), Types: {f64})
610 .legalFor(Pred: ST.hasStdExtZfh(), Types: {f16})
611 .libcallFor(Types: {f32, f64})
612 .libcallFor(Pred: ST.is64Bit(), Types: {f128});
613
614 getActionDefinitionsBuilder(Opcodes: {G_FNEG, G_FABS})
615 .legalFor(Pred: ST.hasStdExtF(), Types: {f32})
616 .legalFor(Pred: ST.hasStdExtD(), Types: {f64})
617 .legalFor(Pred: ST.hasStdExtZfh(), Types: {f16})
618 .lowerFor(Types: {f32, f64, f128});
619
620 getActionDefinitionsBuilder(Opcode: G_FREM)
621 .libcallFor(Types: {f32, f64})
622 .libcallFor(Pred: ST.is64Bit(), Types: {f128})
623 .minScalar(TypeIdx: 0, Ty: f32)
624 .scalarize(TypeIdx: 0);
625
626 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
627 .legalFor(Pred: ST.hasStdExtF(), Types: {{f32, f32}})
628 .legalFor(Pred: ST.hasStdExtD(), Types: {{f64, f64}, {f32, f64}, {f64, f32}})
629 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{f16, f16}, {f16, f32}, {f32, f16}})
630 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{f16, f64}, {f64, f16}})
631 .lower();
632
633 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
634 .legalFor(Pred: ST.hasStdExtD(), Types: {{f32, f64}})
635 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {{f16, f32}})
636 .legalFor(Pred: ST.hasStdExtZfhmin() && ST.hasStdExtD(), Types: {{f16, f64}})
637 .libcallFor(Types: {{f16, f32}, {f16, f64}, {f32, f64}})
638 .libcallFor(Pred: ST.is64Bit(), Types: {{f32, f128}, {f64, f128}});
639 getActionDefinitionsBuilder(Opcode: G_FPEXT)
640 .legalFor(Pred: ST.hasStdExtD(), Types: {{f64, f32}})
641 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {{f32, f16}})
642 .legalFor(Pred: ST.hasStdExtZfhmin() && ST.hasStdExtD(), Types: {{f64, f16}})
643 .libcallFor(Pred: !ST.hasStdExtZfhmin(), Types: {{f32, f16}})
644 .libcallFor(Pred: !(ST.hasStdExtZfhmin() && ST.hasStdExtD()), Types: {{f64, f16}})
645 .libcallFor(Types: {{f64, f32}})
646 .libcallFor(Pred: ST.is64Bit(), Types: {{f128, f32}, {f128, f64}});
647
648 getActionDefinitionsBuilder(Opcode: G_FCMP)
649 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, f32}})
650 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, f64}})
651 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, f16}})
652 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
653 .minScalar(TypeIdx: 1, Ty: f32)
654 .libcallFor(Types: {{sXLen, f32}, {sXLen, f64}})
655 .libcallFor(Pred: ST.is64Bit(), Types: {{sXLen, f128}});
656
657 // TODO: Support vector version of G_IS_FPCLASS.
658 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
659 .customFor(Pred: ST.hasStdExtF(), Types: {{s1, f32}})
660 .customFor(Pred: ST.hasStdExtD(), Types: {{s1, f64}})
661 .customFor(Pred: ST.hasStdExtZfh(), Types: {{s1, f16}})
662 .lower();
663
664 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
665 .legalFor(Pred: ST.hasStdExtF(), Types: {f32})
666 .legalFor(Pred: ST.hasStdExtD(), Types: {f64})
667 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {f16})
668 .lowerFor(Types: {f32, f64, f128, LLT::bfloat16()});
669
670 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
671 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, f32}})
672 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, f64}})
673 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, f16}})
674 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, f32}})
675 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, f64}})
676 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, f16}})
677 .widenScalarToNextPow2(TypeIdx: 0)
678 .minScalar(TypeIdx: 0, Ty: s32)
679 // The magnitude of a half is at most 65504, so with Zfh use fcvt.w[u].h
680 // and extend the i32 result. Otherwise promote the half source to float
681 // (via fcvt.s.h with Zfhmin, __extendhfsf2 without) and use the float
682 // conversion. On RV32, exclude i64 results here so that they get
683 // narrowed to i32 first.
684 .widenScalarIf(
685 Predicate: [=, &ST](const LegalityQuery &Query) {
686 return Query.Types[1] == f16 && !ST.hasStdExtZfh() &&
687 (ST.is64Bit() || Query.Types[0].isScalar(Size: 32));
688 },
689 Mutation: changeTo(TypeIdx: 1, Ty: f32))
690 .libcallFor(Pred: !ST.hasStdExtZfhmin(), Types: {{s64, f16}})
691 .narrowScalarFor(Types: {{s64, f16}}, Mutation: changeTo(TypeIdx: 0, Ty: s32))
692 .libcallFor(Types: {{s32, f32}, {s64, f32}, {s32, f64}, {s64, f64}})
693 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, f128}, {s64, f128}}) // FIXME RV32.
694 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, f32}, {s128, f64}, {s128, f128}});
695
696 // Unlike G_FPTOSI/G_FPTOUI these need no libcalls for out-of-range values:
697 // the RISC-V conversions already saturate. There is no legalFor() because
698 // they do not produce 0 for NaN, which legalizeCustom fixes up with a
699 // compare and a select. The remaining cases fall back to the generic
700 // lowering, which clamps before converting and only handles scalars.
701 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
702 .customFor(Pred: ST.hasStdExtF(), Types: {{sXLen, f32}})
703 .customFor(Pred: ST.hasStdExtD(), Types: {{sXLen, f64}})
704 .customFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, f16}})
705 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, f32}})
706 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, f64}})
707 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, f16}})
708 // f16 extends exactly to f32; converting via f32 avoids needing rules
709 // for every {sXX, f16} combination.
710 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: f16), Mutation: changeTo(TypeIdx: 1, Ty: f32))
711 .lowerIf(Predicate: all(P0: isScalar(TypeIdx: 0), P1: isScalar(TypeIdx: 1)));
712
713 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
714 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, f32}})
715 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, f64}})
716 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, f16}})
717 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, f32}})
718 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, f64}})
719 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, f16}})
720 .minScalar(TypeIdx: 1, Ty: f32)
721 .libcallFor(Types: {{s32, f32},
722 {s64, f32},
723 {s32, f64},
724 {s64, f64},
725 {s32, f128},
726 {s64, f128}});
727
728 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
729 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, f32}})
730 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, f64}})
731 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, f16}})
732 .minScalar(TypeIdx: 0, Ty: sXLen)
733 .minScalar(TypeIdx: 1, Ty: f32)
734 .libcallFor(Types: {{s32, f32},
735 {s64, f32},
736 {s32, f64},
737 {s64, f64},
738 {s32, f128},
739 {s64, f128}});
740
741 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
742 .legalFor(Pred: ST.hasStdExtF(), Types: {{f32, sXLen}})
743 .legalFor(Pred: ST.hasStdExtD(), Types: {{f64, sXLen}})
744 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{f16, sXLen}})
745 .widenScalarToNextPow2(TypeIdx: 1)
746 // Promote to XLen if the operation is legal.
747 .widenScalarIf(
748 Predicate: [=, &ST](const LegalityQuery &Query) {
749 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
750 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
751 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
752 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
753 (ST.hasStdExtZfh() &&
754 Query.Types[0].getSizeInBits() == 16));
755 },
756 Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: sXLen))
757 // Otherwise only promote to s32 since we have si libcalls.
758 .minScalar(TypeIdx: 1, Ty: s32)
759 .libcallFor(Types: {{f32, s32}, {f64, s32}, {f32, s64}, {f64, s64}})
760 .libcallFor(Pred: ST.is64Bit(), Types: {{f128, s32}, {f128, s64}}) // FIXME RV32.
761 .libcallFor(Pred: ST.is64Bit(), Types: {{f32, s128}, {f64, s128}, {f128, s128}});
762
763 // FIXME: We can do custom inline expansion like SelectionDAG.
764 getActionDefinitionsBuilder(Opcodes: {G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
765 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
766 G_INTRINSIC_ROUNDEVEN})
767 .legalFor(Pred: ST.hasStdExtZfa(), Types: {f32})
768 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {f64})
769 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {f16})
770 .libcallFor(Types: {f32, f64})
771 .libcallFor(Pred: ST.is64Bit(), Types: {f128});
772
773 getActionDefinitionsBuilder(Opcodes: {G_FMAXIMUM, G_FMINIMUM})
774 .legalFor(Pred: ST.hasStdExtZfa(), Types: {f32})
775 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {f64})
776 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {f16});
777
778 getActionDefinitionsBuilder(Opcodes: {G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
779 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
780 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
781 G_FTANH, G_FMODF, G_FSINCOS})
782 .libcallFor(Types: {f32, f64})
783 .libcallFor(Pred: ST.is64Bit(), Types: {f128});
784 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
785 .libcallFor(Types: {{f32, s32}, {f64, s32}})
786 .libcallFor(Pred: ST.is64Bit(), Types: {f128, s32});
787
788 getActionDefinitionsBuilder(Opcode: G_FCANONICALIZE)
789 .legalFor(Pred: ST.hasStdExtF(), Types: {f32})
790 .legalFor(Pred: ST.hasStdExtD(), Types: {f64})
791 .legalFor(Pred: ST.hasStdExtZfh(), Types: {f16});
792
793 getActionDefinitionsBuilder(Opcode: G_VASTART).customFor(Types: {p0});
794
795 // va_list must be a pointer, but most sized types are pretty easy to handle
796 // as the destination.
797 getActionDefinitionsBuilder(Opcode: G_VAARG)
798 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types
799 // other than sXLen.
800 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
801 .lowerForCartesianProduct(Types0: {sXLen, p0}, Types1: {p0});
802
803 getActionDefinitionsBuilder(Opcode: G_VSCALE)
804 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
805 .customFor(Types: {sXLen});
806
807 auto &SplatActions =
808 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
809 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
810 P1: typeIs(TypeIdx: 1, TypesInit: sXLen)))
811 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: typeIs(TypeIdx: 1, TypesInit: s1)));
812 // Handle case of s64 element vectors on RV32. If the subtarget does not have
813 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget
814 // does have f64, then we don't know whether the type is an f64 or an i64,
815 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,
816 // depending on how the instructions it consumes are legalized. They are not
817 // legalized yet since legalization is in reverse postorder, so we cannot
818 // make the decision at this moment.
819 if (XLen == 32) {
820 if (ST.hasVInstructionsF64() && ST.hasStdExtD())
821 SplatActions.legalIf(Predicate: all(
822 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
823 else if (ST.hasVInstructionsI64())
824 SplatActions.customIf(Predicate: all(
825 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
826 }
827
828 SplatActions.clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
829
830 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
831 LLT DstTy = Query.Types[0];
832 LLT SrcTy = Query.Types[1];
833 return DstTy.getElementType() == LLT::scalar(SizeInBits: 1) &&
834 DstTy.getElementCount().getKnownMinValue() >= 8 &&
835 SrcTy.getElementCount().getKnownMinValue() >= 8;
836 };
837 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
838 // We don't have the ability to slide mask vectors down indexed by their
839 // i1 elements; the smallest we can do is i8. Often we are able to bitcast
840 // to equivalent i8 vectors.
841 .bitcastIf(
842 Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
843 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST), args: ExtractSubvecBitcastPred),
844 Mutation: [=](const LegalityQuery &Query) {
845 LLT CastTy = LLT::vector(
846 EC: Query.Types[0].getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8);
847 return std::pair(0, CastTy);
848 })
849 .customIf(Predicate: LegalityPredicates::any(
850 P0: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
851 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)),
852 P1: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
853 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST))));
854
855 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR)
856 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
857 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
858 .customIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
859 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)));
860
861 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
862 .lowerIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
863
864 getActionDefinitionsBuilder(Opcodes: {G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
865 G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
866 G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
867 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
868 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
869 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
870
871 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_SUB)
872 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
873 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
874 .lower();
875
876 getActionDefinitionsBuilder(
877 Opcodes: {G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
878 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
879 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
880 .unsupported();
881
882 getActionDefinitionsBuilder(Opcode: G_PREFETCH).legalIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p0));
883
884 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
885 LLT VecTy = Query.Types[0];
886 LLT EltTy = Query.Types[1];
887 return VecTy.getElementType() == EltTy;
888 };
889
890 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
891 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
892 P1: InsertVectorEltPred, args: typeIs(TypeIdx: 2, TypesInit: sXLen)))
893 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: InsertVectorEltPred,
894 args: typeIs(TypeIdx: 2, TypesInit: sXLen)));
895
896 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
897 .alwaysLegal();
898
899 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
900
901 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
902
903 verify(MII: *ST.getInstrInfo());
904}
905
906bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
907 MachineInstr &MI) const {
908 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
909
910 if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
911 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {
912 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {
913 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
914 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
915
916 auto OldScalar = MI.getOperand(i: II->ScalarOperand + 2).getReg();
917 // Legalize integer vx form intrinsic.
918 if (MRI.getType(Reg: OldScalar).isScalar()) {
919 if (MRI.getType(Reg: OldScalar).getSizeInBits() < iXLen.getSizeInBits()) {
920 Helper.Observer.changingInstr(MI);
921 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: II->ScalarOperand + 2,
922 ExtOpcode: TargetOpcode::G_ANYEXT);
923 Helper.Observer.changedInstr(MI);
924 } else if (MRI.getType(Reg: OldScalar).getSizeInBits() >
925 iXLen.getSizeInBits()) {
926 // TODO: i64 in riscv32.
927 return false;
928 }
929 }
930 }
931 return true;
932 }
933
934 switch (IntrinsicID) {
935 default:
936 return false;
937 case Intrinsic::riscv_orc_b: {
938 if (!STI.hasStdExtZbb())
939 return false;
940
941 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
942 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
943 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
944 if (Ty == iXLen)
945 return true;
946 if (!STI.is64Bit() || Ty != LLT::scalar(SizeInBits: 32))
947 return false;
948
949 // Each output byte depends only on the corresponding input byte.
950 auto Src = MIRBuilder.buildAnyExt(Res: iXLen, Op: MI.getOperand(i: 2));
951 auto Result = MIRBuilder.buildIntrinsic(ID: IntrinsicID, Res: {DstOp(iXLen)})
952 .addUse(RegNo: Src.getReg(Idx: 0));
953 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: Result);
954 MI.eraseFromParent();
955 return true;
956 }
957 case Intrinsic::riscv_clmulh:
958 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULH, DstOps: {MI.getOperand(i: 0)},
959 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
960 MI.eraseFromParent();
961 return true;
962 case Intrinsic::riscv_clmulr:
963 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULR, DstOps: {MI.getOperand(i: 0)},
964 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
965 MI.eraseFromParent();
966 return true;
967 case Intrinsic::vacopy: {
968 // vacopy arguments must be legal because of the intrinsic signature.
969 // No need to check here.
970
971 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
972 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
973 MachineFunction &MF = *MI.getMF();
974 const DataLayout &DL = MIRBuilder.getDataLayout();
975 LLVMContext &Ctx = MF.getFunction().getContext();
976
977 Register DstLst = MI.getOperand(i: 1).getReg();
978 LLT PtrTy = MRI.getType(Reg: DstLst);
979
980 // Load the source va_list
981 Align Alignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
982 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
983 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: Alignment);
984 auto Tmp = MIRBuilder.buildLoad(Res: PtrTy, Addr: MI.getOperand(i: 2), MMO&: *LoadMMO);
985
986 // Store the result in the destination va_list
987 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
988 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: Alignment);
989 MIRBuilder.buildStore(Val: Tmp, Addr: DstLst, MMO&: *StoreMMO);
990
991 MI.eraseFromParent();
992 return true;
993 }
994 case Intrinsic::riscv_vsetvli:
995 case Intrinsic::riscv_vsetvlimax:
996 case Intrinsic::riscv_masked_atomicrmw_add:
997 case Intrinsic::riscv_masked_atomicrmw_sub:
998 case Intrinsic::riscv_masked_atomicrmw_xchg:
999 case Intrinsic::riscv_masked_atomicrmw_max:
1000 case Intrinsic::riscv_masked_atomicrmw_min:
1001 case Intrinsic::riscv_masked_atomicrmw_umax:
1002 case Intrinsic::riscv_masked_atomicrmw_umin:
1003 case Intrinsic::riscv_masked_cmpxchg:
1004 return true;
1005 }
1006}
1007
1008bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
1009 MachineIRBuilder &MIRBuilder) const {
1010 // Stores the address of the VarArgsFrameIndex slot into the memory location
1011 assert(MI.getOpcode() == TargetOpcode::G_VASTART);
1012 MachineFunction *MF = MI.getParent()->getParent();
1013 RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>();
1014 int FI = FuncInfo->getVarArgsFrameIndex();
1015 LLT AddrTy = MIRBuilder.getMRI()->getType(Reg: MI.getOperand(i: 0).getReg());
1016 auto FINAddr = MIRBuilder.buildFrameIndex(Res: AddrTy, Idx: FI);
1017 assert(MI.hasOneMemOperand());
1018 MIRBuilder.buildStore(Val: FINAddr, Addr: MI.getOperand(i: 0).getReg(),
1019 MMO&: *MI.memoperands()[0]);
1020 MI.eraseFromParent();
1021 return true;
1022}
1023
1024bool RISCVLegalizerInfo::legalizeReadCounter(
1025 MachineInstr &MI, MachineIRBuilder &MIRBuilder,
1026 GISelChangeObserver &Observer) const {
1027 assert((MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER ||
1028 MI.getOpcode() == TargetOpcode::G_READSTEADYCOUNTER) &&
1029 "Unexpected opcode");
1030 assert(!STI.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
1031 "has custom type legalization on riscv32");
1032
1033 // On RV32 a 64-bit counter CSR must be read as two 32-bit halves. Because
1034 // the count may wrap between the two reads, re-read the high half and loop
1035 // until the two high reads agree.
1036 int64_t LoCounter, HiCounter;
1037 if (MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER) {
1038 LoCounter = RISCVSysReg::cycle;
1039 HiCounter = RISCVSysReg::cycleh;
1040 } else {
1041 LoCounter = RISCVSysReg::time;
1042 HiCounter = RISCVSysReg::timeh;
1043 }
1044
1045 MachineBasicBlock *BB = MI.getParent();
1046 MachineFunction &MF = *BB->getParent();
1047 const BasicBlock *LLVMBB = BB->getBasicBlock();
1048 DebugLoc DL = MI.getDebugLoc();
1049 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1050
1051 // Split BB into an entry that falls through into a loop block, and a done
1052 // block that receives the remainder of BB and its original successors.
1053 MachineFunction::iterator It = std::next(x: BB->getIterator());
1054 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
1055 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
1056 MF.insert(MBBI: It, MBB: LoopMBB);
1057 MF.insert(MBBI: It, MBB: DoneMBB);
1058
1059 // Splice the instructions after the readcyclecounter into DoneMBB, notifying
1060 // the observer about each moved instruction so CSEInfo stays consistent.
1061 for (MachineBasicBlock::iterator I = std::next(x: MI.getIterator()),
1062 E = BB->end();
1063 I != E; ++I)
1064 Observer.changingInstr(MI&: *I);
1065 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
1066 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
1067 for (MachineInstr &MovedMI : DoneMBB->instrs())
1068 Observer.changedInstr(MI&: MovedMI);
1069 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
1070 BB->addSuccessor(Succ: LoopMBB);
1071
1072 LLT i32 = LLT::integer(SizeInBits: 32);
1073 // Generic vregs carry the s32 type for G_MERGE_VALUES below, but are also
1074 // constrained to GPR so the target CSRRS/BNE instructions satisfy the
1075 // verifier's register-class constraints.
1076 auto CreateGPR = [&]() {
1077 Register R = MRI.createGenericVirtualRegister(Ty: i32);
1078 MRI.setRegClass(Reg: R, RC: &RISCV::GPRRegClass);
1079 return R;
1080 };
1081 Register LoReg = CreateGPR();
1082 Register HiReg = CreateGPR();
1083 Register ReadAgainReg = CreateGPR();
1084
1085 // read:
1086 // csrrs HiReg, counterh # high word
1087 // csrrs LoReg, counter # low word
1088 // csrrs ReadAgainReg, counterh
1089 // bne HiReg, ReadAgainReg, read
1090 // Emit the target instructions directly with BuildMI.
1091 const RISCVInstrInfo *TII = STI.getInstrInfo();
1092 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
1093 .addImm(Val: HiCounter)
1094 .addReg(RegNo: RISCV::X0);
1095 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
1096 .addImm(Val: LoCounter)
1097 .addReg(RegNo: RISCV::X0);
1098 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
1099 .addImm(Val: HiCounter)
1100 .addReg(RegNo: RISCV::X0);
1101
1102 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
1103 .addReg(RegNo: HiReg)
1104 .addReg(RegNo: ReadAgainReg)
1105 .addMBB(MBB: LoopMBB);
1106
1107 LoopMBB->addSuccessor(Succ: LoopMBB);
1108 LoopMBB->addSuccessor(Succ: DoneMBB);
1109
1110 // Re-pair the two halves into the 64-bit result.
1111 Register DstReg = MI.getOperand(i: 0).getReg();
1112 Observer.erasingInstr(MI);
1113 MI.eraseFromParent();
1114
1115 MIRBuilder.setInsertPt(MBB&: *DoneMBB, II: DoneMBB->begin());
1116 MIRBuilder.setDebugLoc(DL);
1117 MIRBuilder.buildMergeValues(Res: DstReg, Ops: {LoReg, HiReg});
1118 return true;
1119}
1120
1121bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,
1122 MachineIRBuilder &MIRBuilder) const {
1123 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1124 auto &MF = *MI.getParent()->getParent();
1125 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
1126 unsigned EntrySize = MJTI->getEntrySize(TD: MF.getDataLayout());
1127
1128 Register PtrReg = MI.getOperand(i: 0).getReg();
1129 LLT PtrTy = MRI.getType(Reg: PtrReg);
1130 Register IndexReg = MI.getOperand(i: 2).getReg();
1131 LLT IndexTy = MRI.getType(Reg: IndexReg);
1132
1133 if (!isPowerOf2_32(Value: EntrySize))
1134 return false;
1135
1136 auto ShiftAmt = MIRBuilder.buildConstant(Res: IndexTy, Val: Log2_32(Value: EntrySize));
1137 IndexReg = MIRBuilder.buildShl(Dst: IndexTy, Src0: IndexReg, Src1: ShiftAmt).getReg(Idx: 0);
1138
1139 auto Addr = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: IndexReg);
1140
1141 MachineMemOperand *MMO = MF.getMachineMemOperand(
1142 PtrInfo: MachinePointerInfo::getJumpTable(MF), F: MachineMemOperand::MOLoad,
1143 Size: EntrySize, BaseAlignment: Align(MJTI->getEntryAlignment(TD: MF.getDataLayout())));
1144
1145 Register TargetReg;
1146 switch (MJTI->getEntryKind()) {
1147 default:
1148 return false;
1149 case MachineJumpTableInfo::EK_LabelDifference32: {
1150 // For PIC, the sequence is:
1151 // BRIND(load(Jumptable + index) + RelocBase)
1152 // RelocBase can be JumpTable, GOT or some sort of global base.
1153 unsigned LoadOpc =
1154 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;
1155 auto Load = MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: IndexTy, Addr, MMO&: *MMO);
1156 TargetReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: Load).getReg(Idx: 0);
1157 break;
1158 }
1159 case MachineJumpTableInfo::EK_Custom32: {
1160 auto Load = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: IndexTy,
1161 Addr, MMO&: *MMO);
1162 TargetReg = MIRBuilder.buildIntToPtr(Dst: PtrTy, Src: Load).getReg(Idx: 0);
1163 break;
1164 }
1165 case MachineJumpTableInfo::EK_BlockAddress:
1166 TargetReg = MIRBuilder.buildLoad(Res: PtrTy, Addr, MMO&: *MMO).getReg(Idx: 0);
1167 break;
1168 }
1169
1170 if (MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch()) {
1171 // When cf-protection-branch is enabled, we need to use a software
1172 // guarded branch for jump table branch.
1173 MIRBuilder.buildInstr(Opc: RISCV::G_SW_GUARDED_BRIND, DstOps: {}, SrcOps: {TargetReg});
1174 } else {
1175 MIRBuilder.buildBrIndirect(Tgt: TargetReg);
1176 }
1177
1178 MI.eraseFromParent();
1179 return true;
1180}
1181
1182bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,
1183 bool ShouldOptForSize) const {
1184 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);
1185 int64_t Imm = APImm.getSExtValue();
1186 // All simm32 constants should be handled by isel.
1187 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
1188 // this check redundant, but small immediates are common so this check
1189 // should have better compile time.
1190 if (isInt<32>(x: Imm))
1191 return false;
1192
1193 // We only need to cost the immediate, if constant pool lowering is enabled.
1194 if (!STI.useConstantPoolForLargeInts())
1195 return false;
1196
1197 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI);
1198 if (Seq.size() <= STI.getMaxBuildIntsCost())
1199 return false;
1200
1201 // Optimizations below are disabled for opt size. If we're optimizing for
1202 // size, use a constant pool.
1203 if (ShouldOptForSize)
1204 return true;
1205 //
1206 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
1207 // that if it will avoid a constant pool.
1208 // It will require an extra temporary register though.
1209 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
1210 // low and high 32 bits are the same and bit 31 and 63 are set.
1211 unsigned ShiftAmt, AddOpc;
1212 RISCVMatInt::InstSeq SeqLo =
1213 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI, ShiftAmt, AddOpc);
1214 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
1215}
1216
1217bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
1218 MachineIRBuilder &MIB) const {
1219 Register Dst = MI.getOperand(i: 0).getReg();
1220
1221 // We define our scalable vector types for lmul=1 to use a 64 bit known
1222 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
1223 // vscale as VLENB / 8.
1224 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
1225 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1226 // Support for VLEN==32 is incomplete.
1227 return false;
1228
1229 // We assume VLENB is a multiple of 8. We manually choose the best shift
1230 // here because SimplifyDemandedBits isn't always able to simplify it.
1231 uint64_t Val = MI.getOperand(i: 1).getCImm()->getZExtValue();
1232 if (isPowerOf2_64(Value: Val)) {
1233 uint64_t Log2 = Log2_64(Value: Val);
1234 if (Log2 < 3) {
1235 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {iXLen}, SrcOps: {});
1236 MIB.buildLShr(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: iXLen, Val: 3 - Log2),
1237 Flags: MachineInstr::IsExact);
1238 } else if (Log2 > 3) {
1239 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {iXLen}, SrcOps: {});
1240 MIB.buildShl(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: iXLen, Val: Log2 - 3));
1241 } else {
1242 MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {Dst}, SrcOps: {});
1243 }
1244 } else if ((Val % 8) == 0) {
1245 // If the multiplier is a multiple of 8, scale it down to avoid needing
1246 // to shift the VLENB value.
1247 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {iXLen}, SrcOps: {});
1248 MIB.buildMul(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: iXLen, Val: Val / 8));
1249 } else {
1250 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {iXLen}, SrcOps: {});
1251 auto VScale = MIB.buildLShr(Dst: iXLen, Src0: VLENB, Src1: MIB.buildConstant(Res: iXLen, Val: 3),
1252 Flags: MachineInstr::IsExact);
1253 MIB.buildMul(Dst, Src0: VScale, Src1: MIB.buildConstant(Res: iXLen, Val));
1254 }
1255 MI.eraseFromParent();
1256 return true;
1257}
1258
1259// Custom-lower extensions from mask vectors by using a vselect either with 1
1260// for zero/any-extension or -1 for sign-extension:
1261// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
1262// Note that any-extension is lowered identically to zero-extension.
1263bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,
1264 MachineIRBuilder &MIB) const {
1265
1266 unsigned Opc = MI.getOpcode();
1267 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||
1268 Opc == TargetOpcode::G_ANYEXT);
1269
1270 MachineRegisterInfo &MRI = *MIB.getMRI();
1271 Register Dst = MI.getOperand(i: 0).getReg();
1272 Register Src = MI.getOperand(i: 1).getReg();
1273
1274 LLT DstTy = MRI.getType(Reg: Dst);
1275 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;
1276 LLT DstEltTy = DstTy.getElementType();
1277 auto SplatZero = MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: 0));
1278 auto SplatTrue =
1279 MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: ExtTrueVal));
1280 MIB.buildSelect(Res: Dst, Tst: Src, Op0: SplatTrue, Op1: SplatZero);
1281
1282 MI.eraseFromParent();
1283 return true;
1284}
1285
1286bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,
1287 LegalizerHelper &Helper,
1288 MachineIRBuilder &MIB) const {
1289 assert((isa<GLoad>(MI) || isa<GStore>(MI)) &&
1290 "Machine instructions must be Load/Store.");
1291 MachineRegisterInfo &MRI = *MIB.getMRI();
1292 MachineFunction *MF = MI.getMF();
1293 const DataLayout &DL = MIB.getDataLayout();
1294 LLVMContext &Ctx = MF->getFunction().getContext();
1295
1296 Register DstReg = MI.getOperand(i: 0).getReg();
1297 LLT DataTy = MRI.getType(Reg: DstReg);
1298 if (!DataTy.isVector())
1299 return false;
1300
1301 if (!MI.hasOneMemOperand())
1302 return false;
1303
1304 MachineMemOperand *MMO = *MI.memoperands_begin();
1305
1306 const auto *TLI = STI.getTargetLowering();
1307 EVT VT = EVT::getEVT(Ty: getTypeForLLT(Ty: DataTy, C&: Ctx));
1308
1309 if (TLI->allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT, MMO: *MMO))
1310 return true;
1311
1312 unsigned EltSizeBits = DataTy.getScalarSizeInBits();
1313 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
1314 "Unexpected unaligned RVV load type");
1315
1316 // Calculate the new vector type with i8 elements
1317 unsigned NumElements =
1318 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);
1319 LLT NewDataTy = LLT::scalable_vector(MinNumElements: NumElements, ScalarSizeInBits: 8);
1320
1321 Helper.bitcast(MI, TypeIdx: 0, Ty: NewDataTy);
1322
1323 return true;
1324}
1325
1326/// Return the type of the mask type suitable for masking the provided
1327/// vector type. This is simply an i1 element type vector of the same
1328/// (possibly scalable) length.
1329static LLT getMaskTypeFor(LLT VecTy) {
1330 assert(VecTy.isVector());
1331 ElementCount EC = VecTy.getElementCount();
1332 return LLT::vector(EC, ScalarTy: LLT::integer(SizeInBits: 1));
1333}
1334
1335/// Creates an all ones mask suitable for masking a vector of type VecTy with
1336/// vector length VL.
1337static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,
1338 MachineIRBuilder &MIB,
1339 MachineRegisterInfo &MRI) {
1340 LLT MaskTy = getMaskTypeFor(VecTy);
1341 return MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {MaskTy}, SrcOps: {VL});
1342}
1343
1344/// Gets the two common "VL" operands: an all-ones mask and the vector length.
1345/// VecTy is a scalable vector type.
1346static std::pair<MachineInstrBuilder, MachineInstrBuilder>
1347buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1348 assert(VecTy.isScalableVector() && "Expecting scalable container type");
1349 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
1350 LLT XLenTy(STI.getXLenVT());
1351 auto VL = MIB.buildConstant(Res: XLenTy, Val: -1);
1352 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
1353 return {Mask, VL};
1354}
1355
1356static MachineInstrBuilder
1357buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
1358 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
1359 MachineRegisterInfo &MRI) {
1360 // TODO: If the Hi bits of the splat are undefined, then it's fine to just
1361 // splat Lo even if it might be sign extended. I don't think we have
1362 // introduced a case where we're build a s64 where the upper bits are undef
1363 // yet.
1364
1365 // Fall back to a stack store and stride x0 vector load.
1366 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in
1367 // preprocessDAG in SDAG.
1368 return MIB.buildInstr(Opc: RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, DstOps: {Dst},
1369 SrcOps: {Passthru, Lo, Hi, VL});
1370}
1371
1372static MachineInstrBuilder
1373buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
1374 const SrcOp &Scalar, const SrcOp &VL,
1375 MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1376 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
1377 auto Unmerge = MIB.buildUnmerge(Res: LLT::integer(SizeInBits: 32), Op: Scalar);
1378 return buildSplatPartsS64WithVL(Dst, Passthru, Lo: Unmerge.getReg(Idx: 0),
1379 Hi: Unmerge.getReg(Idx: 1), VL, MIB, MRI);
1380}
1381
1382// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a
1383// legal equivalently-sized i8 type, so we can use that as a go-between.
1384// Splats of s1 types that have constant value can be legalized as VMSET_VL or
1385// VMCLR_VL.
1386bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
1387 MachineIRBuilder &MIB) const {
1388 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);
1389
1390 MachineRegisterInfo &MRI = *MIB.getMRI();
1391
1392 Register Dst = MI.getOperand(i: 0).getReg();
1393 Register SplatVal = MI.getOperand(i: 1).getReg();
1394
1395 LLT VecTy = MRI.getType(Reg: Dst);
1396 LLT XLenTy(STI.getXLenVT());
1397
1398 // Handle case of s64 element vectors on rv32
1399 if (XLenTy.getSizeInBits() == 32 &&
1400 VecTy.getElementType().getSizeInBits() == 64) {
1401 auto [_, VL] = buildDefaultVLOps(VecTy: MRI.getType(Reg: Dst), MIB, MRI);
1402 buildSplatSplitS64WithVL(Dst, Passthru: MIB.buildUndef(Res: VecTy), Scalar: SplatVal, VL, MIB,
1403 MRI);
1404 MI.eraseFromParent();
1405 return true;
1406 }
1407
1408 // All-zeros or all-ones splats are handled specially.
1409 MachineInstr &SplatValMI = *MRI.getVRegDef(Reg: SplatVal);
1410 if (isAllOnesOrAllOnesSplat(MI: SplatValMI, MRI)) {
1411 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1412 MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {Dst}, SrcOps: {VL});
1413 MI.eraseFromParent();
1414 return true;
1415 }
1416 if (isNullOrNullSplat(MI: SplatValMI, MRI)) {
1417 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1418 MIB.buildInstr(Opc: RISCV::G_VMCLR_VL, DstOps: {Dst}, SrcOps: {VL});
1419 MI.eraseFromParent();
1420 return true;
1421 }
1422
1423 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all
1424 // ones) by promoting it to an s8 splat.
1425 LLT InterEltTy = LLT::integer(SizeInBits: 8);
1426 LLT InterTy = VecTy.changeElementType(NewEltTy: InterEltTy);
1427 auto ZExtSplatVal = MIB.buildZExt(Res: InterEltTy, Op: SplatVal);
1428 auto And =
1429 MIB.buildAnd(Dst: InterEltTy, Src0: ZExtSplatVal, Src1: MIB.buildConstant(Res: InterEltTy, Val: 1));
1430 auto LHS = MIB.buildSplatVector(Res: InterTy, Val: And);
1431 auto ZeroSplat =
1432 MIB.buildSplatVector(Res: InterTy, Val: MIB.buildConstant(Res: InterEltTy, Val: 0));
1433 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: LHS, Op1: ZeroSplat);
1434 MI.eraseFromParent();
1435 return true;
1436}
1437
1438static LLT getLMUL1Ty(LLT VecTy) {
1439 assert(VecTy.getElementType().getSizeInBits() <= 64 &&
1440 "Unexpected vector LLT");
1441 return LLT::scalable_vector(MinNumElements: RISCV::RVVBitsPerBlock /
1442 VecTy.getElementType().getSizeInBits(),
1443 ScalarTy: VecTy.getElementType());
1444}
1445
1446bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
1447 MachineIRBuilder &MIB) const {
1448 GExtractSubvector &ES = cast<GExtractSubvector>(Val&: MI);
1449
1450 MachineRegisterInfo &MRI = *MIB.getMRI();
1451
1452 Register Dst = ES.getReg(Idx: 0);
1453 Register Src = ES.getSrcVec();
1454 uint64_t Idx = ES.getIndexImm();
1455
1456 // With an index of 0 this is a cast-like subvector, which can be performed
1457 // with subregister operations.
1458 if (Idx == 0)
1459 return true;
1460
1461 LLT LitTy = MRI.getType(Reg: Dst);
1462 LLT BigTy = MRI.getType(Reg: Src);
1463
1464 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1465 // We can't slide this mask vector up indexed by its i1 elements.
1466 // This poses a problem when we wish to insert a scalable vector which
1467 // can't be re-expressed as a larger type. Just choose the slow path and
1468 // extend to a larger type, then truncate back down.
1469 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 8));
1470 LLT ExtLitTy = LitTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 8));
1471 auto BigZExt = MIB.buildZExt(Res: ExtBigTy, Op: Src);
1472 auto ExtractZExt = MIB.buildExtractSubvector(Res: ExtLitTy, Src: BigZExt, Index: Idx);
1473 auto SplatZero = MIB.buildSplatVector(
1474 Res: ExtLitTy, Val: MIB.buildConstant(Res: ExtLitTy.getElementType(), Val: 0));
1475 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: ExtractZExt, Op1: SplatZero);
1476 MI.eraseFromParent();
1477 return true;
1478 }
1479
1480 // extract_subvector scales the index by vscale if the subvector is scalable,
1481 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
1482 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1483 MVT LitTyMVT = getMVTForLLT(Ty: LitTy);
1484 auto Decompose =
1485 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1486 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: LitTyMVT, InsertExtractIdx: Idx, TRI);
1487 unsigned RemIdx = Decompose.second;
1488
1489 // If the Idx has been completely eliminated then this is a subvector extract
1490 // which naturally aligns to a vector register. These can easily be handled
1491 // using subregister manipulation.
1492 if (RemIdx == 0)
1493 return true;
1494
1495 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
1496 // was > M1 then the index would need to be a multiple of VLMAX, and so would
1497 // divide exactly.
1498 assert(
1499 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||
1500 RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVVType::LMUL_1);
1501
1502 // If the vector type is an LMUL-group type, extract a subvector equal to the
1503 // nearest full vector register type.
1504 LLT InterLitTy = BigTy;
1505 Register Vec = Src;
1506 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1507 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1508 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
1509 // we should have successfully decomposed the extract into a subregister.
1510 assert(Decompose.first != RISCV::NoSubRegister);
1511 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1512 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
1513 // specified on the source Register (the equivalent) since generic virtual
1514 // register does not allow subregister index.
1515 Vec = MIB.buildExtractSubvector(Res: InterLitTy, Src, Index: Idx - RemIdx).getReg(Idx: 0);
1516 }
1517
1518 // Slide this vector register down by the desired number of elements in order
1519 // to place the desired subvector starting at element 0.
1520 const LLT XLenTy(STI.getXLenVT());
1521 auto SlidedownAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1522 auto [Mask, VL] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1523 uint64_t Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
1524 auto Slidedown = MIB.buildInstr(
1525 Opc: RISCV::G_VSLIDEDOWN_VL, DstOps: {InterLitTy},
1526 SrcOps: {MIB.buildUndef(Res: InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
1527
1528 // Now the vector is in the right position, extract our final subvector. This
1529 // should resolve to a COPY.
1530 MIB.buildExtractSubvector(Res: Dst, Src: Slidedown, Index: 0);
1531
1532 MI.eraseFromParent();
1533 return true;
1534}
1535
1536bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,
1537 LegalizerHelper &Helper,
1538 MachineIRBuilder &MIB) const {
1539 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
1540
1541 MachineRegisterInfo &MRI = *MIB.getMRI();
1542
1543 Register Dst = IS.getReg(Idx: 0);
1544 Register BigVec = IS.getBigVec();
1545 Register LitVec = IS.getSubVec();
1546 uint64_t Idx = IS.getIndexImm();
1547
1548 LLT BigTy = MRI.getType(Reg: BigVec);
1549 LLT LitTy = MRI.getType(Reg: LitVec);
1550
1551 if (Idx == 0 && mi_match(R: BigVec, MRI, P: m_GImplicitDef()))
1552 return true;
1553
1554 // We don't have the ability to slide mask vectors up indexed by their i1
1555 // elements; the smallest we can do is i8. Often we are able to bitcast to
1556 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i8
1557 // vectors and truncate down after the insert.
1558 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1559 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();
1560 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();
1561 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)
1562 return Helper.bitcast(
1563 MI&: IS, TypeIdx: 0,
1564 Ty: LLT::vector(EC: BigTy.getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8));
1565
1566 // We can't slide this mask vector up indexed by its i1 elements.
1567 // This poses a problem when we wish to insert a scalable vector which
1568 // can't be re-expressed as a larger type. Just choose the slow path and
1569 // extend to a larger type, then truncate back down.
1570 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::integer(SizeInBits: 8));
1571 return Helper.widenScalar(MI&: IS, TypeIdx: 0, WideTy: ExtBigTy);
1572 }
1573
1574 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1575 unsigned SubRegIdx, RemIdx;
1576 std::tie(args&: SubRegIdx, args&: RemIdx) =
1577 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1578 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: getMVTForLLT(Ty: LitTy), InsertExtractIdx: Idx, TRI);
1579
1580 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
1581 assert(isPowerOf2_64(
1582 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));
1583 bool ExactlyVecRegSized =
1584 STI.expandVScale(X: LitTy.getSizeInBits())
1585 .isKnownMultipleOf(RHS: STI.expandVScale(X: VecRegSize));
1586
1587 // If the Idx has been completely eliminated and this subvector's size is a
1588 // vector register or a multiple thereof, or the surrounding elements are
1589 // undef, then this is a subvector insert which naturally aligns to a vector
1590 // register. These can easily be handled using subregister manipulation.
1591 if (RemIdx == 0 && ExactlyVecRegSized)
1592 return true;
1593
1594 // If the subvector is smaller than a vector register, then the insertion
1595 // must preserve the undisturbed elements of the register. We do this by
1596 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
1597 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
1598 // subvector within the vector register, and an INSERT_SUBVECTOR of that
1599 // LMUL=1 type back into the larger vector (resolving to another subregister
1600 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
1601 // to avoid allocating a large register group to hold our subvector.
1602
1603 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
1604 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
1605 // (in our case undisturbed). This means we can set up a subvector insertion
1606 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
1607 // size of the subvector.
1608 const LLT XLenTy(STI.getXLenVT());
1609 LLT InterLitTy = BigTy;
1610 Register AlignedExtract = BigVec;
1611 unsigned AlignedIdx = Idx - RemIdx;
1612 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1613 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1614 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1615 // Extract a subvector equal to the nearest full vector register type. This
1616 // should resolve to a G_EXTRACT on a subreg.
1617 AlignedExtract =
1618 MIB.buildExtractSubvector(Res: InterLitTy, Src: BigVec, Index: AlignedIdx).getReg(Idx: 0);
1619 }
1620
1621 auto Insert = MIB.buildInsertSubvector(Res: InterLitTy, Src0: MIB.buildUndef(Res: InterLitTy),
1622 Src1: LitVec, Index: 0);
1623
1624 auto [Mask, _] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1625 auto VL = MIB.buildVScale(Res: XLenTy, MinElts: LitTy.getElementCount().getKnownMinValue());
1626
1627 // If we're inserting into the lowest elements, use a tail undisturbed
1628 // vmv.v.v.
1629 MachineInstrBuilder Inserted;
1630 bool NeedInsertSubvec =
1631 TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(), RHS: InterLitTy.getSizeInBits());
1632 Register InsertedDst =
1633 NeedInsertSubvec ? MRI.createGenericVirtualRegister(Ty: InterLitTy) : Dst;
1634 if (RemIdx == 0) {
1635 Inserted = MIB.buildInstr(Opc: RISCV::G_VMV_V_V_VL, DstOps: {InsertedDst},
1636 SrcOps: {AlignedExtract, Insert, VL});
1637 } else {
1638 auto SlideupAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1639 // Construct the vector length corresponding to RemIdx + length(LitTy).
1640 VL = MIB.buildAdd(Dst: XLenTy, Src0: SlideupAmt, Src1: VL);
1641 // Use tail agnostic policy if we're inserting over InterLitTy's tail.
1642 ElementCount EndIndex =
1643 ElementCount::getScalable(MinVal: RemIdx) + LitTy.getElementCount();
1644 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
1645 if (STI.expandVScale(X: EndIndex) ==
1646 STI.expandVScale(X: InterLitTy.getElementCount()))
1647 Policy = RISCVVType::TAIL_AGNOSTIC;
1648
1649 Inserted =
1650 MIB.buildInstr(Opc: RISCV::G_VSLIDEUP_VL, DstOps: {InsertedDst},
1651 SrcOps: {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});
1652 }
1653
1654 // If required, insert this subvector back into the correct vector register.
1655 // This should resolve to an INSERT_SUBREG instruction.
1656 if (NeedInsertSubvec)
1657 MIB.buildInsertSubvector(Res: Dst, Src0: BigVec, Src1: Inserted, Index: AlignedIdx);
1658
1659 MI.eraseFromParent();
1660 return true;
1661}
1662
1663bool RISCVLegalizerInfo::legalizeBitreverse(MachineInstr &MI,
1664 MachineIRBuilder &MIB) const {
1665 assert(MI.getOpcode() == TargetOpcode::G_BITREVERSE && "Unexpected opcode");
1666
1667 if (!STI.hasStdExtZbkb())
1668 return false;
1669
1670 MachineRegisterInfo &MRI = *MIB.getMRI();
1671
1672 Register Dst = MI.getOperand(i: 0).getReg();
1673 Register Src = MI.getOperand(i: 1).getReg();
1674
1675 if (!MRI.getType(Reg: Dst).isScalar(Size: 8))
1676 return false;
1677
1678 auto WideSrc = MIB.buildAnyExt(Res: iXLen, Op: Src);
1679 auto Brev = MIB.buildInstr(Opc: RISCV::G_BREV8, DstOps: {iXLen}, SrcOps: {WideSrc.getReg(Idx: 0)});
1680 MIB.buildTrunc(Res: Dst, Op: Brev.getReg(Idx: 0));
1681
1682 MI.eraseFromParent();
1683 return true;
1684}
1685
1686static unsigned getRISCVWOpcode(unsigned Opcode) {
1687 switch (Opcode) {
1688 default:
1689 llvm_unreachable("Unexpected opcode");
1690 case TargetOpcode::G_ASHR:
1691 return RISCV::G_SRAW;
1692 case TargetOpcode::G_LSHR:
1693 return RISCV::G_SRLW;
1694 case TargetOpcode::G_SHL:
1695 return RISCV::G_SLLW;
1696 case TargetOpcode::G_SDIV:
1697 return RISCV::G_DIVW;
1698 case TargetOpcode::G_UDIV:
1699 return RISCV::G_DIVUW;
1700 case TargetOpcode::G_UREM:
1701 return RISCV::G_REMUW;
1702 case TargetOpcode::G_ROTL:
1703 return RISCV::G_ROLW;
1704 case TargetOpcode::G_ROTR:
1705 return RISCV::G_RORW;
1706 case TargetOpcode::G_CTLZ:
1707 return RISCV::G_CLZW;
1708 case TargetOpcode::G_CTTZ:
1709 return RISCV::G_CTZW;
1710 case TargetOpcode::G_CTLS:
1711 return RISCV::G_CLSW;
1712 case TargetOpcode::G_FPTOSI:
1713 return RISCV::G_FCVT_W_RV64;
1714 case TargetOpcode::G_FPTOUI:
1715 return RISCV::G_FCVT_WU_RV64;
1716 }
1717}
1718
1719bool RISCVLegalizerInfo::legalizeCustom(
1720 LegalizerHelper &Helper, MachineInstr &MI,
1721 LostDebugLocObserver &LocObserver) const {
1722 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1723 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1724 MachineFunction &MF = *MI.getParent()->getParent();
1725 switch (MI.getOpcode()) {
1726 default:
1727 // No idea what to do.
1728 return false;
1729 case TargetOpcode::G_ABS:
1730 return Helper.lowerAbsToMaxNeg(MI);
1731 case TargetOpcode::G_CLMULH:
1732 case TargetOpcode::G_CLMULR: {
1733 assert(STI.is64Bit() &&
1734 MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32) &&
1735 "Unexpected custom legalization");
1736 // Shift both inputs by 32 so the full product has 64 trailing zeros.
1737 // Perform CLMULH or CLMULR on the shifted inputs, then extract the upper
1738 // 32 bits of the result.
1739 auto Shift = MIRBuilder.buildConstant(Res: iXLen, Val: 32);
1740 auto LHS = MIRBuilder.buildAnyExt(Res: iXLen, Op: MI.getOperand(i: 1));
1741 auto RHS = MIRBuilder.buildAnyExt(Res: iXLen, Op: MI.getOperand(i: 2));
1742 auto ShiftedLHS = MIRBuilder.buildShl(Dst: iXLen, Src0: LHS, Src1: Shift);
1743 auto ShiftedRHS = MIRBuilder.buildShl(Dst: iXLen, Src0: RHS, Src1: Shift);
1744 auto Product = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {iXLen},
1745 SrcOps: {ShiftedLHS, ShiftedRHS});
1746 auto High = MIRBuilder.buildLShr(Dst: iXLen, Src0: Product, Src1: Shift);
1747 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: High);
1748 MI.eraseFromParent();
1749 return true;
1750 }
1751 case TargetOpcode::G_CONSTANT: {
1752 const Function &F = MF.getFunction();
1753 // TODO: if PSI and BFI are present, add " ||
1754 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".
1755 bool ShouldOptForSize = F.hasOptSize();
1756 const ConstantInt *ConstVal = MI.getOperand(i: 1).getCImm();
1757 if (!shouldBeInConstantPool(APImm: ConstVal->getValue(), ShouldOptForSize))
1758 return true;
1759 return Helper.lowerConstant(MI);
1760 }
1761 case TargetOpcode::G_SUB:
1762 case TargetOpcode::G_ADD: {
1763 Helper.Observer.changingInstr(MI);
1764 // The G_ANYEXTs below leave the new high bits unconstrained, so no-wrap
1765 // claims proved at the narrow width no longer hold.
1766 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap);
1767 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1768 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1769
1770 Register DstALU = MRI.createGenericVirtualRegister(Ty: iXLen);
1771
1772 MachineOperand &MO = MI.getOperand(i: 0);
1773 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
1774 auto DstSext = MIRBuilder.buildSExtInReg(Res: iXLen, Op: DstALU, ImmOp: 32);
1775
1776 MIRBuilder.buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: {MO}, SrcOps: {DstSext});
1777 MO.setReg(DstALU);
1778
1779 Helper.Observer.changedInstr(MI);
1780 return true;
1781 }
1782 case TargetOpcode::G_ASHR:
1783 case TargetOpcode::G_LSHR:
1784 case TargetOpcode::G_SHL: {
1785 if (getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI)) {
1786 // We don't need a custom node for shift by constant. Just widen the
1787 // source and the shift amount.
1788 unsigned ExtOpc = TargetOpcode::G_ANYEXT;
1789 if (MI.getOpcode() == TargetOpcode::G_ASHR)
1790 ExtOpc = TargetOpcode::G_SEXT;
1791 else if (MI.getOpcode() == TargetOpcode::G_LSHR)
1792 ExtOpc = TargetOpcode::G_ZEXT;
1793
1794 Helper.Observer.changingInstr(MI);
1795 // The G_ANYEXT leave the new high bits unconstrained, so no-wrap
1796 // claims proved at the narrow width no longer hold.
1797 if (ExtOpc == TargetOpcode::G_ANYEXT)
1798 MI.clearFlags(flags: MachineInstr::NoUWrap | MachineInstr::NoSWrap);
1799 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 1, ExtOpcode: ExtOpc);
1800 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
1801 Helper.widenScalarDst(MI, WideTy: iXLen);
1802 Helper.Observer.changedInstr(MI);
1803 return true;
1804 }
1805
1806 Helper.Observer.changingInstr(MI);
1807 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1808 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1809 Helper.widenScalarDst(MI, WideTy: iXLen);
1810 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1811 Helper.Observer.changedInstr(MI);
1812 return true;
1813 }
1814 case TargetOpcode::G_SDIV:
1815 case TargetOpcode::G_UDIV:
1816 case TargetOpcode::G_UREM:
1817 case TargetOpcode::G_ROTL:
1818 case TargetOpcode::G_ROTR: {
1819 Helper.Observer.changingInstr(MI);
1820 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1821 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1822 Helper.widenScalarDst(MI, WideTy: iXLen);
1823 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1824 Helper.Observer.changedInstr(MI);
1825 return true;
1826 }
1827 case TargetOpcode::G_CTLZ:
1828 case TargetOpcode::G_CTTZ:
1829 case TargetOpcode::G_CTLS: {
1830 Helper.Observer.changingInstr(MI);
1831 Helper.widenScalarSrc(MI, WideTy: iXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1832 Helper.widenScalarDst(MI, WideTy: iXLen);
1833 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1834 Helper.Observer.changedInstr(MI);
1835 return true;
1836 }
1837 case TargetOpcode::G_FPTOSI:
1838 case TargetOpcode::G_FPTOUI: {
1839 Helper.Observer.changingInstr(MI);
1840 Helper.widenScalarDst(MI, WideTy: iXLen);
1841 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1842 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RTZ));
1843 Helper.Observer.changedInstr(MI);
1844 return true;
1845 }
1846 case TargetOpcode::G_FPTOSI_SAT:
1847 case TargetOpcode::G_FPTOUI_SAT: {
1848 // Mirror lowerFP_TO_INT_SAT (RISCVISelLowering.cpp): the conversion
1849 // saturates out-of-range values, but we need to select 0 for NaN as
1850 // required by the intrinsics.
1851 Register Dst = MI.getOperand(i: 0).getReg();
1852 Register Src = MI.getOperand(i: 1).getReg();
1853 LLT DstTy = MRI.getType(Reg: Dst);
1854 unsigned DstBits = DstTy.getScalarSizeInBits();
1855 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
1856
1857 unsigned FCvtOp;
1858 if (DstBits == XLen) {
1859 FCvtOp = IsSigned ? RISCV::G_FCVT_X : RISCV::G_FCVT_XU;
1860 } else {
1861 assert(STI.is64Bit() && "Non-XLen results are only custom on RV64");
1862 FCvtOp = IsSigned ? RISCV::G_FCVT_W_RV64 : RISCV::G_FCVT_WU_RV64;
1863 }
1864
1865 auto FCvt = MIRBuilder.buildInstr(Opc: FCvtOp, DstOps: {iXLen}, SrcOps: {Src})
1866 .addImm(Val: RISCVFPRndMode::RTZ);
1867
1868 // RISC-V G_FCMP results are XLen wide, and so are G_SELECT conditions.
1869 auto IsNaN = MIRBuilder.buildFCmp(Pred: CmpInst::FCMP_UNO, Res: iXLen, Op0: Src, Op1: Src);
1870
1871 Register FCvtReg = FCvt.getReg(Idx: 0);
1872 if (DstBits != XLen)
1873 FCvtReg = MIRBuilder.buildTrunc(Res: DstTy, Op: FCvtReg).getReg(Idx: 0);
1874
1875 MIRBuilder.buildSelect(Res: Dst, Tst: IsNaN, Op0: MIRBuilder.buildConstant(Res: DstTy, Val: 0),
1876 Op1: FCvtReg);
1877
1878 MI.eraseFromParent();
1879 return true;
1880 }
1881 case TargetOpcode::G_LROUND: {
1882 // The (i32 any_lround) Pat is IsRV32-only; on RV64 lower to
1883 // riscv_fcvt_w_rv64 with FRM_RMM.
1884 Helper.Observer.changingInstr(MI);
1885 Helper.widenScalarDst(MI, WideTy: iXLen);
1886 MI.setDesc(MIRBuilder.getTII().get(Opcode: RISCV::G_FCVT_W_RV64));
1887 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RMM));
1888 Helper.Observer.changedInstr(MI);
1889 return true;
1890 }
1891 case TargetOpcode::G_READCYCLECOUNTER:
1892 case TargetOpcode::G_READSTEADYCOUNTER:
1893 return legalizeReadCounter(MI, MIRBuilder, Observer&: Helper.Observer);
1894 case TargetOpcode::G_IS_FPCLASS: {
1895 Register GISFPCLASS = MI.getOperand(i: 0).getReg();
1896 Register Src = MI.getOperand(i: 1).getReg();
1897 const MachineOperand &ImmOp = MI.getOperand(i: 2);
1898 MachineIRBuilder MIB(MI);
1899
1900 // Turn LLVM IR's floating point classes to that in RISC-V,
1901 // by simply rotating the 10-bit immediate right by two bits.
1902 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));
1903 auto FClassMask = MIB.buildConstant(Res: iXLen, Val: GFpClassImm.rotr(rotateAmt: 2).zext(width: XLen));
1904 auto ConstZero = MIB.buildConstant(Res: iXLen, Val: 0);
1905
1906 auto GFClass = MIB.buildInstr(Opc: RISCV::G_FCLASS, DstOps: {iXLen}, SrcOps: {Src});
1907 auto And = MIB.buildAnd(Dst: iXLen, Src0: GFClass, Src1: FClassMask);
1908 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: GISFPCLASS, Op0: And, Op1: ConstZero);
1909
1910 MI.eraseFromParent();
1911 return true;
1912 }
1913 case TargetOpcode::G_BRJT:
1914 return legalizeBRJT(MI, MIRBuilder);
1915 case TargetOpcode::G_VASTART:
1916 return legalizeVAStart(MI, MIRBuilder);
1917 case TargetOpcode::G_VSCALE:
1918 return legalizeVScale(MI, MIB&: MIRBuilder);
1919 case TargetOpcode::G_ZEXT:
1920 case TargetOpcode::G_SEXT:
1921 case TargetOpcode::G_ANYEXT:
1922 return legalizeExt(MI, MIB&: MIRBuilder);
1923 case TargetOpcode::G_SPLAT_VECTOR:
1924 return legalizeSplatVector(MI, MIB&: MIRBuilder);
1925 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1926 return legalizeExtractSubvector(MI, MIB&: MIRBuilder);
1927 case TargetOpcode::G_INSERT_SUBVECTOR:
1928 return legalizeInsertSubvector(MI, Helper, MIB&: MIRBuilder);
1929 case TargetOpcode::G_BITREVERSE:
1930 return legalizeBitreverse(MI, MIB&: MIRBuilder);
1931 case TargetOpcode::G_LOAD:
1932 case TargetOpcode::G_STORE:
1933 return legalizeLoadStore(MI, Helper, MIB&: MIRBuilder);
1934 }
1935
1936 llvm_unreachable("expected switch to return");
1937}
1938