1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for RISC-V.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "RISCVLegalizerInfo.h"
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVSubtarget.h"
17#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/MachineConstantPool.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/TargetOpcodes.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsRISCV.h"
32#include "llvm/IR/Type.h"
33
34using namespace llvm;
35using namespace LegalityPredicates;
36using namespace LegalizeMutations;
37using namespace MIPatternMatch;
38
39static LegalityPredicate
40typeIsLegalIntOrFPVec(unsigned TypeIdx,
41 std::initializer_list<LLT> IntOrFPVecTys,
42 const RISCVSubtarget &ST) {
43 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
44 return ST.hasVInstructions() &&
45 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
46 ST.hasVInstructionsI64()) &&
47 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
48 ST.getELen() == 64);
49 };
50
51 return all(P0: typeInSet(TypeIdx, TypesInit: IntOrFPVecTys), P1: P);
52}
53
54static LegalityPredicate
55typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
56 const RISCVSubtarget &ST) {
57 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
58 return ST.hasVInstructions() &&
59 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
60 ST.getELen() == 64);
61 };
62 return all(P0: typeInSet(TypeIdx, TypesInit: BoolVecTys), P1: P);
63}
64
65static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
66 std::initializer_list<LLT> PtrVecTys,
67 const RISCVSubtarget &ST) {
68 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
69 return ST.hasVInstructions() &&
70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
71 ST.getELen() == 64) &&
72 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
73 Query.Types[TypeIdx].getScalarSizeInBits() == 32);
74 };
75 return all(P0: typeInSet(TypeIdx, TypesInit: PtrVecTys), P1: P);
76}
77
78RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
79 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(SizeInBits: XLen)) {
80 const LLT sDoubleXLen = LLT::scalar(SizeInBits: 2 * XLen);
81 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: XLen);
82 const LLT s1 = LLT::scalar(SizeInBits: 1);
83 const LLT s8 = LLT::scalar(SizeInBits: 8);
84 const LLT s16 = LLT::scalar(SizeInBits: 16);
85 const LLT s32 = LLT::scalar(SizeInBits: 32);
86 const LLT s64 = LLT::scalar(SizeInBits: 64);
87 const LLT s128 = LLT::scalar(SizeInBits: 128);
88
89 const LLT nxv1s1 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s1);
90 const LLT nxv2s1 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s1);
91 const LLT nxv4s1 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s1);
92 const LLT nxv8s1 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s1);
93 const LLT nxv16s1 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s1);
94 const LLT nxv32s1 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s1);
95 const LLT nxv64s1 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s1);
96
97 const LLT nxv1s8 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s8);
98 const LLT nxv2s8 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s8);
99 const LLT nxv4s8 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s8);
100 const LLT nxv8s8 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s8);
101 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
102 const LLT nxv32s8 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s8);
103 const LLT nxv64s8 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s8);
104
105 const LLT nxv1s16 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s16);
106 const LLT nxv2s16 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s16);
107 const LLT nxv4s16 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s16);
108 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
109 const LLT nxv16s16 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s16);
110 const LLT nxv32s16 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s16);
111
112 const LLT nxv1s32 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s32);
113 const LLT nxv2s32 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s32);
114 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
115 const LLT nxv8s32 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s32);
116 const LLT nxv16s32 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s32);
117
118 const LLT nxv1s64 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s64);
119 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
120 const LLT nxv4s64 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s64);
121 const LLT nxv8s64 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s64);
122
123 const LLT nxv1p0 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: p0);
124 const LLT nxv2p0 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: p0);
125 const LLT nxv4p0 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: p0);
126 const LLT nxv8p0 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: p0);
127 const LLT nxv16p0 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: p0);
128
129 using namespace TargetOpcode;
130
131 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
132
133 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,
134 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
135 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
136 nxv1s64, nxv2s64, nxv4s64, nxv8s64};
137
138 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
139
140 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
141 .legalFor(Types: {sXLen})
142 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
143 .customFor(Pred: ST.is64Bit(), Types: {s32})
144 .widenScalarToNextPow2(TypeIdx: 0)
145 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
146
147 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
148 .legalFor(Types: {sXLen})
149 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
150 .widenScalarToNextPow2(TypeIdx: 0)
151 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
152
153 getActionDefinitionsBuilder(
154 Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO, G_READ_REGISTER, G_WRITE_REGISTER})
155 .lower();
156
157 getActionDefinitionsBuilder(Opcodes: {G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})
158 .minScalar(TypeIdx: 0, Ty: sXLen)
159 .lower();
160
161 // TODO: Use Vector Single-Width Saturating Instructions for vector types.
162 getActionDefinitionsBuilder(
163 Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})
164 .lower();
165
166 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
167 .legalFor(Types: {{sXLen, sXLen}})
168 .customFor(Pred: ST.is64Bit(), Types: {{s32, s32}})
169 .widenScalarToNextPow2(TypeIdx: 0)
170 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
171 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
172
173 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
174 .legalFor(Types: {{s32, s16}})
175 .legalFor(Pred: ST.is64Bit(), Types: {{s64, s16}, {s64, s32}})
176 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
177 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
178 .customIf(Predicate: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))
179 .maxScalar(TypeIdx: 0, Ty: sXLen);
180
181 getActionDefinitionsBuilder(Opcode: G_TRUNC).alwaysLegal();
182
183 {
184 LegalityPredicate ValidSextInRegWidth = all(P0: sizeIs(TypeIdx: 0, Size: 64), P1: immIs(ImmIdx: 0, Imm: 32));
185
186 if (STI.hasStdExtZbb())
187 ValidSextInRegWidth =
188 LegalityPredicates::any(P0: ValidSextInRegWidth, P1: immInSet(ImmIdx: 0, ImmsInit: {8, 16}));
189
190 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
191 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: sXLen), P1: ValidSextInRegWidth))
192 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
193 .lower();
194 }
195
196 // Merge/Unmerge
197 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
198 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Opcode: Op);
199 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
200 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
201 if (XLen == 32 && ST.hasStdExtD()) {
202 MergeUnmergeActions.legalIf(
203 Predicate: all(P0: typeIs(TypeIdx: BigTyIdx, TypesInit: s64), P1: typeIs(TypeIdx: LitTyIdx, TypesInit: s32)));
204 }
205 MergeUnmergeActions.widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: XLen)
206 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: XLen)
207 .clampScalar(TypeIdx: LitTyIdx, MinTy: sXLen, MaxTy: sXLen)
208 .clampScalar(TypeIdx: BigTyIdx, MinTy: sXLen, MaxTy: sXLen);
209 }
210
211 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR}).lower();
212
213 getActionDefinitionsBuilder(Opcodes: {G_ROTR, G_ROTL})
214 .legalFor(Pred: ST.hasStdExtZbb() || ST.hasStdExtZbkb(), Types: {{sXLen, sXLen}})
215 .customFor(Pred: ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),
216 Types: {{s32, s32}})
217 .lower();
218
219 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
220 .customFor(Pred: ST.hasStdExtZbkb(), Types: {s8})
221 .maxScalar(TypeIdx: 0, Ty: sXLen)
222 .lower();
223
224 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
225 Predicate: all(P0: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
226 P1: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST)),
227 P1: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST),
228 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))));
229
230 auto &BSWAPActions = getActionDefinitionsBuilder(Opcode: G_BSWAP);
231 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
232 BSWAPActions.legalFor(Types: {sXLen}).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
233 else
234 BSWAPActions.maxScalar(TypeIdx: 0, Ty: sXLen).lower();
235
236 auto &CountZerosActions = getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTTZ});
237 auto &CountZerosPoisonActions =
238 getActionDefinitionsBuilder(Opcodes: {G_CTLZ_ZERO_POISON, G_CTTZ_ZERO_POISON});
239 if (ST.hasStdExtZbb()) {
240 CountZerosActions.legalFor(Types: {{sXLen, sXLen}})
241 .customFor(Types: {{s32, s32}})
242 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
243 .widenScalarToNextPow2(TypeIdx: 0)
244 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
245 } else {
246 CountZerosActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
247 CountZerosPoisonActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
248 }
249 CountZerosPoisonActions.lower();
250
251 auto &CountSignActions = getActionDefinitionsBuilder(Opcode: G_CTLS);
252 if (ST.hasStdExtP()) {
253 CountSignActions.legalFor(Types: {{sXLen, sXLen}})
254 .customFor(Types: {{s32, s32}})
255 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
256 .widenScalarToNextPow2(TypeIdx: 0)
257 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
258 } else {
259 CountSignActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
260 }
261
262 auto &CTPOPActions = getActionDefinitionsBuilder(Opcode: G_CTPOP);
263 if (ST.hasStdExtZbb()) {
264 CTPOPActions.legalFor(Types: {{sXLen, sXLen}})
265 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
266 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
267 } else {
268 CTPOPActions.widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8)
269 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sXLen)
270 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0)
271 .lower();
272 }
273
274 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
275 .legalFor(Types: {p0})
276 .legalFor(Pred: !ST.is64Bit(), Types: {s32})
277 .customFor(Pred: ST.is64Bit(), Types: {s64})
278 .widenScalarToNextPow2(TypeIdx: 0)
279 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
280
281 // TODO: transform illegal vector types into legal vector type
282 getActionDefinitionsBuilder(Opcode: G_FREEZE)
283 .legalFor(Types: {s16, s32, p0})
284 .legalFor(Pred: ST.is64Bit(), Types: {s64})
285 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
286 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
287 .widenScalarToNextPow2(TypeIdx: 0)
288 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen);
289
290 // TODO: transform illegal vector types into legal vector type
291 // TODO: Merge with G_FREEZE?
292 getActionDefinitionsBuilder(
293 Opcodes: {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})
294 .legalFor(Types: {s32, sXLen, p0})
295 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
296 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
297 .widenScalarToNextPow2(TypeIdx: 0)
298 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
299
300 getActionDefinitionsBuilder(Opcode: G_ICMP)
301 .legalFor(Types: {{sXLen, sXLen}, {sXLen, p0}})
302 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
303 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
304 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, MinSize: 8)
305 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
306 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
307
308 getActionDefinitionsBuilder(Opcode: G_SELECT)
309 .legalFor(Types: {{s32, sXLen}, {p0, sXLen}})
310 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
311 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
312 .legalFor(Pred: XLen == 64 || ST.hasStdExtD(), Types: {{s64, sXLen}})
313 .widenScalarToNextPow2(TypeIdx: 0)
314 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)
315 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
316
317 auto &LoadActions = getActionDefinitionsBuilder(Opcode: G_LOAD);
318 auto &StoreActions = getActionDefinitionsBuilder(Opcode: G_STORE);
319 auto &ExtLoadActions = getActionDefinitionsBuilder(Opcodes: {G_SEXTLOAD, G_ZEXTLOAD});
320
321 // Return the alignment needed for scalar memory ops. If unaligned scalar mem
322 // is supported, we only require byte alignment. Otherwise, we need the memory
323 // op to be natively aligned.
324 auto getScalarMemAlign = [&ST](unsigned Size) {
325 return ST.enableUnalignedScalarMem() ? 8 : Size;
326 };
327
328 LoadActions.legalForTypesWithMemDesc(
329 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
330 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
331 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
332 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
333 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
334 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
335 StoreActions.legalForTypesWithMemDesc(
336 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
337 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
338 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
339 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
340 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
341 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
342 ExtLoadActions.legalForTypesWithMemDesc(
343 TypesAndMemDesc: {{.Type0: sXLen, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
344 {.Type0: sXLen, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)}});
345 if (XLen == 64) {
346 LoadActions.legalForTypesWithMemDesc(
347 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
348 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
349 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
350 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
351 StoreActions.legalForTypesWithMemDesc(
352 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
353 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
354 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
355 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
356 ExtLoadActions.legalForTypesWithMemDesc(
357 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)}});
358 } else if (ST.hasStdExtD()) {
359 LoadActions.legalForTypesWithMemDesc(
360 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
361 StoreActions.legalForTypesWithMemDesc(
362 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
363 }
364
365 // Vector loads/stores.
366 if (ST.hasVInstructions()) {
367 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
368 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
369 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
370 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
371 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
372 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
373 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
374 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
375 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
376 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
377 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
378 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
379 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
380 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
381 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
382 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
383 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
384 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
385 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
386 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
387 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
388 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
389 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
390 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
391 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
392 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
393 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
394 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
395 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
396 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
397
398 if (ST.getELen() == 64) {
399 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
400 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
401 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
402 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
403 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
404 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
405 }
406
407 if (ST.hasVInstructionsI64()) {
408 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
409 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
410 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
411 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
412 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
413 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
414 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
415 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
416 }
417
418 // we will take the custom lowering logic if we have scalable vector types
419 // with non-standard alignments
420 LoadActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
421 StoreActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
422
423 // Pointers require that XLen sized elements are legal.
424 if (XLen <= ST.getELen()) {
425 LoadActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
426 StoreActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
427 }
428 }
429
430 LoadActions.widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
431 .lowerIfMemSizeNotByteSizePow2()
432 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
433 .lower();
434 StoreActions
435 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
436 .lowerIfMemSizeNotByteSizePow2()
437 .lower();
438
439 ExtLoadActions.widenScalarToNextPow2(TypeIdx: 0).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).lower();
440
441 getActionDefinitionsBuilder(Opcodes: {G_PTR_ADD, G_PTRMASK}).legalFor(Types: {{p0, sXLen}});
442
443 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
444 .legalFor(Types: {{sXLen, p0}})
445 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
446
447 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
448 .legalFor(Types: {{p0, sXLen}})
449 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
450
451 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
452
453 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {sXLen}).minScalar(TypeIdx: 0, Ty: sXLen);
454
455 getActionDefinitionsBuilder(Opcode: G_BRJT).customFor(Types: {{p0, sXLen}});
456
457 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
458
459 getActionDefinitionsBuilder(Opcode: G_PHI)
460 .legalFor(Types: {p0, s32, sXLen})
461 .widenScalarToNextPow2(TypeIdx: 0)
462 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
463
464 getActionDefinitionsBuilder(Opcodes: {G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
465 .legalFor(Types: {p0});
466
467 if (ST.hasStdExtZmmul()) {
468 getActionDefinitionsBuilder(Opcode: G_MUL)
469 .legalFor(Types: {sXLen})
470 .widenScalarToNextPow2(TypeIdx: 0)
471 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
472
473 // clang-format off
474 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
475 .legalFor(Types: {sXLen})
476 .lower();
477 // clang-format on
478
479 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO}).minScalar(TypeIdx: 0, Ty: sXLen).lower();
480 } else {
481 getActionDefinitionsBuilder(Opcode: G_MUL)
482 .libcallFor(Types: {sXLen, sDoubleXLen})
483 .widenScalarToNextPow2(TypeIdx: 0)
484 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen);
485
486 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).lowerFor(Types: {sXLen});
487
488 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
489 .minScalar(TypeIdx: 0, Ty: sXLen)
490 // Widen sXLen to sDoubleXLen so we can use a single libcall to get
491 // the low bits for the mul result and high bits to do the overflow
492 // check.
493 .widenScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: sXLen),
494 Mutation: LegalizeMutations::changeTo(TypeIdx: 0, Ty: sDoubleXLen))
495 .lower();
496 }
497
498 if (ST.hasStdExtM()) {
499 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV, G_UREM})
500 .legalFor(Types: {sXLen})
501 .customFor(Types: {s32})
502 .libcallFor(Types: {sDoubleXLen})
503 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sDoubleXLen)
504 .widenScalarToNextPow2(TypeIdx: 0);
505 getActionDefinitionsBuilder(Opcode: G_SREM)
506 .legalFor(Types: {sXLen})
507 .libcallFor(Types: {sDoubleXLen})
508 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
509 .widenScalarToNextPow2(TypeIdx: 0);
510 } else {
511 getActionDefinitionsBuilder(Opcodes: {G_UDIV, G_SDIV, G_UREM, G_SREM})
512 .libcallFor(Types: {sXLen, sDoubleXLen})
513 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
514 .widenScalarToNextPow2(TypeIdx: 0);
515 }
516
517 // TODO: Use libcall for sDoubleXLen.
518 getActionDefinitionsBuilder(Opcodes: {G_SDIVREM, G_UDIVREM}).lower();
519
520 getActionDefinitionsBuilder(Opcode: G_ABS)
521 .customFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
522 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
523 .lower();
524
525 getActionDefinitionsBuilder(Opcodes: {G_ABDS, G_ABDU})
526 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
527 .lower();
528
529 getActionDefinitionsBuilder(Opcodes: {G_UMAX, G_UMIN, G_SMAX, G_SMIN})
530 .legalFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
531 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
532 .lower();
533
534 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
535
536 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
537
538 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
539
540 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
541
542 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
543 .lower();
544
545 // On RV64 the 64-bit counter CSRs (cycle/time) are read directly. On RV32
546 // they are custom-legally lowered to a re-read-the-high-half loop (see
547 // legalizeReadCounter).
548 getActionDefinitionsBuilder(Opcodes: {G_READCYCLECOUNTER, G_READSTEADYCOUNTER})
549 .legalFor(Pred: ST.is64Bit(), Types: {s64})
550 .customFor(Pred: !ST.is64Bit(), Types: {s64});
551
552 // FP Operations
553
554 // FIXME: Support s128 for rv32 when libcall handling is able to use sret.
555 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,
556 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,
557 G_FMINIMUMNUM})
558 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
559 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
560 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
561 .libcallFor(Types: {s32, s64})
562 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
563
564 getActionDefinitionsBuilder(Opcodes: {G_FNEG, G_FABS})
565 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
566 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
567 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
568 .lowerFor(Types: {s32, s64, s128});
569
570 getActionDefinitionsBuilder(Opcode: G_FREM)
571 .libcallFor(Types: {s32, s64})
572 .libcallFor(Pred: ST.is64Bit(), Types: {s128})
573 .minScalar(TypeIdx: 0, Ty: s32)
574 .scalarize(TypeIdx: 0);
575
576 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
577 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, s32}})
578 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s64}, {s32, s64}, {s64, s32}})
579 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s16}, {s16, s32}, {s32, s16}})
580 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}, {s64, s16}})
581 .lower();
582
583 // FIXME: Use Zfhmin.
584 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
585 .legalFor(Pred: ST.hasStdExtD(), Types: {{s32, s64}})
586 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s32}})
587 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}})
588 .libcallFor(Types: {{s32, s64}})
589 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}});
590 getActionDefinitionsBuilder(Opcode: G_FPEXT)
591 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s32}})
592 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s32, s16}})
593 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s64, s16}})
594 .libcallFor(Types: {{s64, s32}})
595 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}});
596
597 getActionDefinitionsBuilder(Opcode: G_FCMP)
598 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
599 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
600 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
601 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
602 .libcallFor(Types: {{sXLen, s32}, {sXLen, s64}})
603 .libcallFor(Pred: ST.is64Bit(), Types: {{sXLen, s128}});
604
605 // TODO: Support vector version of G_IS_FPCLASS.
606 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
607 .customFor(Pred: ST.hasStdExtF(), Types: {{s1, s32}})
608 .customFor(Pred: ST.hasStdExtD(), Types: {{s1, s64}})
609 .customFor(Pred: ST.hasStdExtZfh(), Types: {{s1, s16}})
610 .lower();
611
612 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
613 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
614 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
615 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
616 .customFor(Pred: !ST.is64Bit(), Types: {s32})
617 .customFor(Pred: ST.is64Bit(), Types: {s32, s64})
618 .lowerFor(Types: {s64, s128});
619
620 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
621 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
622 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
623 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
624 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
625 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
626 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
627 .widenScalarToNextPow2(TypeIdx: 0)
628 .minScalar(TypeIdx: 0, Ty: s32)
629 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
630 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}}) // FIXME RV32.
631 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}, {s128, s128}});
632
633 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
634 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
635 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
636 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
637 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
638 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
639 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
640 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16), Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: s32))
641 .libcallFor(Types: {{s32, s32},
642 {s64, s32},
643 {s32, s64},
644 {s64, s64},
645 {s32, s128},
646 {s64, s128}});
647
648 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
649 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, sXLen}})
650 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, sXLen}})
651 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, sXLen}})
652 .widenScalarToNextPow2(TypeIdx: 1)
653 // Promote to XLen if the operation is legal.
654 .widenScalarIf(
655 Predicate: [=, &ST](const LegalityQuery &Query) {
656 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
657 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
658 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
659 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
660 (ST.hasStdExtZfh() &&
661 Query.Types[0].getSizeInBits() == 16));
662 },
663 Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: sXLen))
664 // Otherwise only promote to s32 since we have si libcalls.
665 .minScalar(TypeIdx: 1, Ty: s32)
666 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
667 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}}) // FIXME RV32.
668 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}, {s128, s128}});
669
670 // FIXME: We can do custom inline expansion like SelectionDAG.
671 getActionDefinitionsBuilder(Opcodes: {G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
672 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
673 G_INTRINSIC_ROUNDEVEN})
674 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
675 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
676 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16})
677 .libcallFor(Types: {s32, s64})
678 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
679
680 getActionDefinitionsBuilder(Opcodes: {G_FMAXIMUM, G_FMINIMUM})
681 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
682 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
683 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16});
684
685 getActionDefinitionsBuilder(Opcodes: {G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
686 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
687 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
688 G_FTANH, G_FMODF})
689 .libcallFor(Types: {s32, s64})
690 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
691 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
692 .libcallFor(Types: {{s32, s32}, {s64, s32}})
693 .libcallFor(Pred: ST.is64Bit(), Types: {s128, s32});
694
695 getActionDefinitionsBuilder(Opcode: G_FCANONICALIZE)
696 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
697 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
698 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16});
699
700 getActionDefinitionsBuilder(Opcode: G_VASTART).customFor(Types: {p0});
701
702 // va_list must be a pointer, but most sized types are pretty easy to handle
703 // as the destination.
704 getActionDefinitionsBuilder(Opcode: G_VAARG)
705 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types
706 // other than sXLen.
707 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
708 .lowerForCartesianProduct(Types0: {sXLen, p0}, Types1: {p0});
709
710 getActionDefinitionsBuilder(Opcode: G_VSCALE)
711 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
712 .customFor(Types: {sXLen});
713
714 auto &SplatActions =
715 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
716 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
717 P1: typeIs(TypeIdx: 1, TypesInit: sXLen)))
718 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: typeIs(TypeIdx: 1, TypesInit: s1)));
719 // Handle case of s64 element vectors on RV32. If the subtarget does not have
720 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget
721 // does have f64, then we don't know whether the type is an f64 or an i64,
722 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,
723 // depending on how the instructions it consumes are legalized. They are not
724 // legalized yet since legalization is in reverse postorder, so we cannot
725 // make the decision at this moment.
726 if (XLen == 32) {
727 if (ST.hasVInstructionsF64() && ST.hasStdExtD())
728 SplatActions.legalIf(Predicate: all(
729 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
730 else if (ST.hasVInstructionsI64())
731 SplatActions.customIf(Predicate: all(
732 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
733 }
734
735 SplatActions.clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
736
737 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
738 LLT DstTy = Query.Types[0];
739 LLT SrcTy = Query.Types[1];
740 return DstTy.getElementType() == LLT::scalar(SizeInBits: 1) &&
741 DstTy.getElementCount().getKnownMinValue() >= 8 &&
742 SrcTy.getElementCount().getKnownMinValue() >= 8;
743 };
744 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
745 // We don't have the ability to slide mask vectors down indexed by their
746 // i1 elements; the smallest we can do is i8. Often we are able to bitcast
747 // to equivalent i8 vectors.
748 .bitcastIf(
749 Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
750 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST), args: ExtractSubvecBitcastPred),
751 Mutation: [=](const LegalityQuery &Query) {
752 LLT CastTy = LLT::vector(
753 EC: Query.Types[0].getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8);
754 return std::pair(0, CastTy);
755 })
756 .customIf(Predicate: LegalityPredicates::any(
757 P0: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
758 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)),
759 P1: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
760 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST))));
761
762 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR)
763 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
764 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
765 .customIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
766 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)));
767
768 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
769 .lowerIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
770
771 getActionDefinitionsBuilder(Opcodes: {G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
772 G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
773 G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
774 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
775 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
776 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
777
778 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_SUB)
779 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
780 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
781 .lower();
782
783 getActionDefinitionsBuilder(
784 Opcodes: {G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
785 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
786 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
787 .unsupported();
788
789 getActionDefinitionsBuilder(Opcode: G_PREFETCH).legalIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p0));
790
791 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
792 LLT VecTy = Query.Types[0];
793 LLT EltTy = Query.Types[1];
794 return VecTy.getElementType() == EltTy;
795 };
796
797 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
798 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
799 P1: InsertVectorEltPred, args: typeIs(TypeIdx: 2, TypesInit: sXLen)))
800 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: InsertVectorEltPred,
801 args: typeIs(TypeIdx: 2, TypesInit: sXLen)));
802
803 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
804 .alwaysLegal();
805
806 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
807
808 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
809
810 verify(MII: *ST.getInstrInfo());
811}
812
813bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
814 MachineInstr &MI) const {
815 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
816
817 if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
818 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {
819 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {
820 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
821 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
822
823 auto OldScalar = MI.getOperand(i: II->ScalarOperand + 2).getReg();
824 // Legalize integer vx form intrinsic.
825 if (MRI.getType(Reg: OldScalar).isScalar()) {
826 if (MRI.getType(Reg: OldScalar).getSizeInBits() < sXLen.getSizeInBits()) {
827 Helper.Observer.changingInstr(MI);
828 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: II->ScalarOperand + 2,
829 ExtOpcode: TargetOpcode::G_ANYEXT);
830 Helper.Observer.changedInstr(MI);
831 } else if (MRI.getType(Reg: OldScalar).getSizeInBits() >
832 sXLen.getSizeInBits()) {
833 // TODO: i64 in riscv32.
834 return false;
835 }
836 }
837 }
838 return true;
839 }
840
841 switch (IntrinsicID) {
842 default:
843 return false;
844 case Intrinsic::vacopy: {
845 // vacopy arguments must be legal because of the intrinsic signature.
846 // No need to check here.
847
848 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
849 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
850 MachineFunction &MF = *MI.getMF();
851 const DataLayout &DL = MIRBuilder.getDataLayout();
852 LLVMContext &Ctx = MF.getFunction().getContext();
853
854 Register DstLst = MI.getOperand(i: 1).getReg();
855 LLT PtrTy = MRI.getType(Reg: DstLst);
856
857 // Load the source va_list
858 Align Alignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
859 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
860 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: Alignment);
861 auto Tmp = MIRBuilder.buildLoad(Res: PtrTy, Addr: MI.getOperand(i: 2), MMO&: *LoadMMO);
862
863 // Store the result in the destination va_list
864 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
865 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: Alignment);
866 MIRBuilder.buildStore(Val: Tmp, Addr: DstLst, MMO&: *StoreMMO);
867
868 MI.eraseFromParent();
869 return true;
870 }
871 case Intrinsic::riscv_vsetvli:
872 case Intrinsic::riscv_vsetvlimax:
873 case Intrinsic::riscv_masked_atomicrmw_add:
874 case Intrinsic::riscv_masked_atomicrmw_sub:
875 case Intrinsic::riscv_masked_atomicrmw_xchg:
876 case Intrinsic::riscv_masked_atomicrmw_max:
877 case Intrinsic::riscv_masked_atomicrmw_min:
878 case Intrinsic::riscv_masked_atomicrmw_umax:
879 case Intrinsic::riscv_masked_atomicrmw_umin:
880 case Intrinsic::riscv_masked_cmpxchg:
881 return true;
882 }
883}
884
885bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
886 MachineIRBuilder &MIRBuilder) const {
887 // Stores the address of the VarArgsFrameIndex slot into the memory location
888 assert(MI.getOpcode() == TargetOpcode::G_VASTART);
889 MachineFunction *MF = MI.getParent()->getParent();
890 RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>();
891 int FI = FuncInfo->getVarArgsFrameIndex();
892 LLT AddrTy = MIRBuilder.getMRI()->getType(Reg: MI.getOperand(i: 0).getReg());
893 auto FINAddr = MIRBuilder.buildFrameIndex(Res: AddrTy, Idx: FI);
894 assert(MI.hasOneMemOperand());
895 MIRBuilder.buildStore(Val: FINAddr, Addr: MI.getOperand(i: 0).getReg(),
896 MMO&: *MI.memoperands()[0]);
897 MI.eraseFromParent();
898 return true;
899}
900
901bool RISCVLegalizerInfo::legalizeReadCounter(
902 MachineInstr &MI, MachineIRBuilder &MIRBuilder,
903 GISelChangeObserver &Observer) const {
904 assert((MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER ||
905 MI.getOpcode() == TargetOpcode::G_READSTEADYCOUNTER) &&
906 "Unexpected opcode");
907 assert(!STI.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
908 "has custom type legalization on riscv32");
909
910 // On RV32 a 64-bit counter CSR must be read as two 32-bit halves. Because
911 // the count may wrap between the two reads, re-read the high half and loop
912 // until the two high reads agree.
913 int64_t LoCounter, HiCounter;
914 if (MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER) {
915 LoCounter = RISCVSysReg::cycle;
916 HiCounter = RISCVSysReg::cycleh;
917 } else {
918 LoCounter = RISCVSysReg::time;
919 HiCounter = RISCVSysReg::timeh;
920 }
921
922 MachineBasicBlock *BB = MI.getParent();
923 MachineFunction &MF = *BB->getParent();
924 const BasicBlock *LLVMBB = BB->getBasicBlock();
925 DebugLoc DL = MI.getDebugLoc();
926 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
927
928 // Split BB into an entry that falls through into a loop block, and a done
929 // block that receives the remainder of BB and its original successors.
930 MachineFunction::iterator It = std::next(x: BB->getIterator());
931 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
932 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
933 MF.insert(MBBI: It, MBB: LoopMBB);
934 MF.insert(MBBI: It, MBB: DoneMBB);
935
936 // Splice the instructions after the readcyclecounter into DoneMBB, notifying
937 // the observer about each moved instruction so CSEInfo stays consistent.
938 for (MachineBasicBlock::iterator I = std::next(x: MI.getIterator()),
939 E = BB->end();
940 I != E; ++I)
941 Observer.changingInstr(MI&: *I);
942 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
943 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
944 for (MachineInstr &MovedMI : DoneMBB->instrs())
945 Observer.changedInstr(MI&: MovedMI);
946 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
947 BB->addSuccessor(Succ: LoopMBB);
948
949 LLT S32 = LLT::scalar(SizeInBits: 32);
950 // Generic vregs carry the s32 type for G_MERGE_VALUES below, but are also
951 // constrained to GPR so the target CSRRS/BNE instructions satisfy the
952 // verifier's register-class constraints.
953 auto CreateGPR = [&]() {
954 Register R = MRI.createGenericVirtualRegister(Ty: S32);
955 MRI.setRegClass(Reg: R, RC: &RISCV::GPRRegClass);
956 return R;
957 };
958 Register LoReg = CreateGPR();
959 Register HiReg = CreateGPR();
960 Register ReadAgainReg = CreateGPR();
961
962 // read:
963 // csrrs HiReg, counterh # high word
964 // csrrs LoReg, counter # low word
965 // csrrs ReadAgainReg, counterh
966 // bne HiReg, ReadAgainReg, read
967 // Emit the target instructions directly with BuildMI.
968 const RISCVInstrInfo *TII = STI.getInstrInfo();
969 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
970 .addImm(Val: HiCounter)
971 .addReg(RegNo: RISCV::X0);
972 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
973 .addImm(Val: LoCounter)
974 .addReg(RegNo: RISCV::X0);
975 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
976 .addImm(Val: HiCounter)
977 .addReg(RegNo: RISCV::X0);
978
979 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
980 .addReg(RegNo: HiReg)
981 .addReg(RegNo: ReadAgainReg)
982 .addMBB(MBB: LoopMBB);
983
984 LoopMBB->addSuccessor(Succ: LoopMBB);
985 LoopMBB->addSuccessor(Succ: DoneMBB);
986
987 // Re-pair the two halves into the 64-bit result.
988 Register DstReg = MI.getOperand(i: 0).getReg();
989 Observer.erasingInstr(MI);
990 MI.eraseFromParent();
991
992 MIRBuilder.setInsertPt(MBB&: *DoneMBB, II: DoneMBB->begin());
993 MIRBuilder.setDebugLoc(DL);
994 MIRBuilder.buildMergeValues(Res: DstReg, Ops: {LoReg, HiReg});
995 return true;
996}
997
998bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,
999 MachineIRBuilder &MIRBuilder) const {
1000 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1001 auto &MF = *MI.getParent()->getParent();
1002 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
1003 unsigned EntrySize = MJTI->getEntrySize(TD: MF.getDataLayout());
1004
1005 Register PtrReg = MI.getOperand(i: 0).getReg();
1006 LLT PtrTy = MRI.getType(Reg: PtrReg);
1007 Register IndexReg = MI.getOperand(i: 2).getReg();
1008 LLT IndexTy = MRI.getType(Reg: IndexReg);
1009
1010 if (!isPowerOf2_32(Value: EntrySize))
1011 return false;
1012
1013 auto ShiftAmt = MIRBuilder.buildConstant(Res: IndexTy, Val: Log2_32(Value: EntrySize));
1014 IndexReg = MIRBuilder.buildShl(Dst: IndexTy, Src0: IndexReg, Src1: ShiftAmt).getReg(Idx: 0);
1015
1016 auto Addr = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: IndexReg);
1017
1018 MachineMemOperand *MMO = MF.getMachineMemOperand(
1019 PtrInfo: MachinePointerInfo::getJumpTable(MF), F: MachineMemOperand::MOLoad,
1020 Size: EntrySize, BaseAlignment: Align(MJTI->getEntryAlignment(TD: MF.getDataLayout())));
1021
1022 Register TargetReg;
1023 switch (MJTI->getEntryKind()) {
1024 default:
1025 return false;
1026 case MachineJumpTableInfo::EK_LabelDifference32: {
1027 // For PIC, the sequence is:
1028 // BRIND(load(Jumptable + index) + RelocBase)
1029 // RelocBase can be JumpTable, GOT or some sort of global base.
1030 unsigned LoadOpc =
1031 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;
1032 auto Load = MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: IndexTy, Addr, MMO&: *MMO);
1033 TargetReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: Load).getReg(Idx: 0);
1034 break;
1035 }
1036 case MachineJumpTableInfo::EK_Custom32: {
1037 auto Load = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: IndexTy,
1038 Addr, MMO&: *MMO);
1039 TargetReg = MIRBuilder.buildIntToPtr(Dst: PtrTy, Src: Load).getReg(Idx: 0);
1040 break;
1041 }
1042 case MachineJumpTableInfo::EK_BlockAddress:
1043 TargetReg = MIRBuilder.buildLoad(Res: PtrTy, Addr, MMO&: *MMO).getReg(Idx: 0);
1044 break;
1045 }
1046
1047 MIRBuilder.buildBrIndirect(Tgt: TargetReg);
1048
1049 MI.eraseFromParent();
1050 return true;
1051}
1052
1053bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,
1054 bool ShouldOptForSize) const {
1055 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);
1056 int64_t Imm = APImm.getSExtValue();
1057 // All simm32 constants should be handled by isel.
1058 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
1059 // this check redundant, but small immediates are common so this check
1060 // should have better compile time.
1061 if (isInt<32>(x: Imm))
1062 return false;
1063
1064 // We only need to cost the immediate, if constant pool lowering is enabled.
1065 if (!STI.useConstantPoolForLargeInts())
1066 return false;
1067
1068 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI);
1069 if (Seq.size() <= STI.getMaxBuildIntsCost())
1070 return false;
1071
1072 // Optimizations below are disabled for opt size. If we're optimizing for
1073 // size, use a constant pool.
1074 if (ShouldOptForSize)
1075 return true;
1076 //
1077 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
1078 // that if it will avoid a constant pool.
1079 // It will require an extra temporary register though.
1080 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
1081 // low and high 32 bits are the same and bit 31 and 63 are set.
1082 unsigned ShiftAmt, AddOpc;
1083 RISCVMatInt::InstSeq SeqLo =
1084 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI, ShiftAmt, AddOpc);
1085 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
1086}
1087
1088bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
1089 MachineIRBuilder &MIB) const {
1090 Register Dst = MI.getOperand(i: 0).getReg();
1091
1092 // We define our scalable vector types for lmul=1 to use a 64 bit known
1093 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
1094 // vscale as VLENB / 8.
1095 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
1096 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1097 // Support for VLEN==32 is incomplete.
1098 return false;
1099
1100 // We assume VLENB is a multiple of 8. We manually choose the best shift
1101 // here because SimplifyDemandedBits isn't always able to simplify it.
1102 uint64_t Val = MI.getOperand(i: 1).getCImm()->getZExtValue();
1103 if (isPowerOf2_64(Value: Val)) {
1104 uint64_t Log2 = Log2_64(Value: Val);
1105 if (Log2 < 3) {
1106 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1107 MIB.buildLShr(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3 - Log2),
1108 Flags: MachineInstr::IsExact);
1109 } else if (Log2 > 3) {
1110 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1111 MIB.buildShl(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Log2 - 3));
1112 } else {
1113 MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {Dst}, SrcOps: {});
1114 }
1115 } else if ((Val % 8) == 0) {
1116 // If the multiplier is a multiple of 8, scale it down to avoid needing
1117 // to shift the VLENB value.
1118 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1119 MIB.buildMul(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Val / 8));
1120 } else {
1121 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1122 auto VScale = MIB.buildLShr(Dst: sXLen, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3),
1123 Flags: MachineInstr::IsExact);
1124 MIB.buildMul(Dst, Src0: VScale, Src1: MIB.buildConstant(Res: sXLen, Val));
1125 }
1126 MI.eraseFromParent();
1127 return true;
1128}
1129
1130// Custom-lower extensions from mask vectors by using a vselect either with 1
1131// for zero/any-extension or -1 for sign-extension:
1132// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
1133// Note that any-extension is lowered identically to zero-extension.
1134bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,
1135 MachineIRBuilder &MIB) const {
1136
1137 unsigned Opc = MI.getOpcode();
1138 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||
1139 Opc == TargetOpcode::G_ANYEXT);
1140
1141 MachineRegisterInfo &MRI = *MIB.getMRI();
1142 Register Dst = MI.getOperand(i: 0).getReg();
1143 Register Src = MI.getOperand(i: 1).getReg();
1144
1145 LLT DstTy = MRI.getType(Reg: Dst);
1146 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;
1147 LLT DstEltTy = DstTy.getElementType();
1148 auto SplatZero = MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: 0));
1149 auto SplatTrue =
1150 MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: ExtTrueVal));
1151 MIB.buildSelect(Res: Dst, Tst: Src, Op0: SplatTrue, Op1: SplatZero);
1152
1153 MI.eraseFromParent();
1154 return true;
1155}
1156
1157bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,
1158 LegalizerHelper &Helper,
1159 MachineIRBuilder &MIB) const {
1160 assert((isa<GLoad>(MI) || isa<GStore>(MI)) &&
1161 "Machine instructions must be Load/Store.");
1162 MachineRegisterInfo &MRI = *MIB.getMRI();
1163 MachineFunction *MF = MI.getMF();
1164 const DataLayout &DL = MIB.getDataLayout();
1165 LLVMContext &Ctx = MF->getFunction().getContext();
1166
1167 Register DstReg = MI.getOperand(i: 0).getReg();
1168 LLT DataTy = MRI.getType(Reg: DstReg);
1169 if (!DataTy.isVector())
1170 return false;
1171
1172 if (!MI.hasOneMemOperand())
1173 return false;
1174
1175 MachineMemOperand *MMO = *MI.memoperands_begin();
1176
1177 const auto *TLI = STI.getTargetLowering();
1178 EVT VT = EVT::getEVT(Ty: getTypeForLLT(Ty: DataTy, C&: Ctx));
1179
1180 if (TLI->allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT, MMO: *MMO))
1181 return true;
1182
1183 unsigned EltSizeBits = DataTy.getScalarSizeInBits();
1184 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
1185 "Unexpected unaligned RVV load type");
1186
1187 // Calculate the new vector type with i8 elements
1188 unsigned NumElements =
1189 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);
1190 LLT NewDataTy = LLT::scalable_vector(MinNumElements: NumElements, ScalarSizeInBits: 8);
1191
1192 Helper.bitcast(MI, TypeIdx: 0, Ty: NewDataTy);
1193
1194 return true;
1195}
1196
1197/// Return the type of the mask type suitable for masking the provided
1198/// vector type. This is simply an i1 element type vector of the same
1199/// (possibly scalable) length.
1200static LLT getMaskTypeFor(LLT VecTy) {
1201 assert(VecTy.isVector());
1202 ElementCount EC = VecTy.getElementCount();
1203 return LLT::vector(EC, ScalarTy: LLT::scalar(SizeInBits: 1));
1204}
1205
1206/// Creates an all ones mask suitable for masking a vector of type VecTy with
1207/// vector length VL.
1208static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,
1209 MachineIRBuilder &MIB,
1210 MachineRegisterInfo &MRI) {
1211 LLT MaskTy = getMaskTypeFor(VecTy);
1212 return MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {MaskTy}, SrcOps: {VL});
1213}
1214
1215/// Gets the two common "VL" operands: an all-ones mask and the vector length.
1216/// VecTy is a scalable vector type.
1217static std::pair<MachineInstrBuilder, MachineInstrBuilder>
1218buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1219 assert(VecTy.isScalableVector() && "Expecting scalable container type");
1220 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
1221 LLT XLenTy(STI.getXLenVT());
1222 auto VL = MIB.buildConstant(Res: XLenTy, Val: -1);
1223 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
1224 return {Mask, VL};
1225}
1226
1227static MachineInstrBuilder
1228buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
1229 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
1230 MachineRegisterInfo &MRI) {
1231 // TODO: If the Hi bits of the splat are undefined, then it's fine to just
1232 // splat Lo even if it might be sign extended. I don't think we have
1233 // introduced a case where we're build a s64 where the upper bits are undef
1234 // yet.
1235
1236 // Fall back to a stack store and stride x0 vector load.
1237 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in
1238 // preprocessDAG in SDAG.
1239 return MIB.buildInstr(Opc: RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, DstOps: {Dst},
1240 SrcOps: {Passthru, Lo, Hi, VL});
1241}
1242
1243static MachineInstrBuilder
1244buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
1245 const SrcOp &Scalar, const SrcOp &VL,
1246 MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1247 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
1248 auto Unmerge = MIB.buildUnmerge(Res: LLT::scalar(SizeInBits: 32), Op: Scalar);
1249 return buildSplatPartsS64WithVL(Dst, Passthru, Lo: Unmerge.getReg(Idx: 0),
1250 Hi: Unmerge.getReg(Idx: 1), VL, MIB, MRI);
1251}
1252
1253// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a
1254// legal equivalently-sized i8 type, so we can use that as a go-between.
1255// Splats of s1 types that have constant value can be legalized as VMSET_VL or
1256// VMCLR_VL.
1257bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
1258 MachineIRBuilder &MIB) const {
1259 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);
1260
1261 MachineRegisterInfo &MRI = *MIB.getMRI();
1262
1263 Register Dst = MI.getOperand(i: 0).getReg();
1264 Register SplatVal = MI.getOperand(i: 1).getReg();
1265
1266 LLT VecTy = MRI.getType(Reg: Dst);
1267 LLT XLenTy(STI.getXLenVT());
1268
1269 // Handle case of s64 element vectors on rv32
1270 if (XLenTy.getSizeInBits() == 32 &&
1271 VecTy.getElementType().getSizeInBits() == 64) {
1272 auto [_, VL] = buildDefaultVLOps(VecTy: MRI.getType(Reg: Dst), MIB, MRI);
1273 buildSplatSplitS64WithVL(Dst, Passthru: MIB.buildUndef(Res: VecTy), Scalar: SplatVal, VL, MIB,
1274 MRI);
1275 MI.eraseFromParent();
1276 return true;
1277 }
1278
1279 // All-zeros or all-ones splats are handled specially.
1280 MachineInstr &SplatValMI = *MRI.getVRegDef(Reg: SplatVal);
1281 if (isAllOnesOrAllOnesSplat(MI: SplatValMI, MRI)) {
1282 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1283 MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {Dst}, SrcOps: {VL});
1284 MI.eraseFromParent();
1285 return true;
1286 }
1287 if (isNullOrNullSplat(MI: SplatValMI, MRI)) {
1288 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1289 MIB.buildInstr(Opc: RISCV::G_VMCLR_VL, DstOps: {Dst}, SrcOps: {VL});
1290 MI.eraseFromParent();
1291 return true;
1292 }
1293
1294 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all
1295 // ones) by promoting it to an s8 splat.
1296 LLT InterEltTy = LLT::scalar(SizeInBits: 8);
1297 LLT InterTy = VecTy.changeElementType(NewEltTy: InterEltTy);
1298 auto ZExtSplatVal = MIB.buildZExt(Res: InterEltTy, Op: SplatVal);
1299 auto And =
1300 MIB.buildAnd(Dst: InterEltTy, Src0: ZExtSplatVal, Src1: MIB.buildConstant(Res: InterEltTy, Val: 1));
1301 auto LHS = MIB.buildSplatVector(Res: InterTy, Val: And);
1302 auto ZeroSplat =
1303 MIB.buildSplatVector(Res: InterTy, Val: MIB.buildConstant(Res: InterEltTy, Val: 0));
1304 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: LHS, Op1: ZeroSplat);
1305 MI.eraseFromParent();
1306 return true;
1307}
1308
1309static LLT getLMUL1Ty(LLT VecTy) {
1310 assert(VecTy.getElementType().getSizeInBits() <= 64 &&
1311 "Unexpected vector LLT");
1312 return LLT::scalable_vector(MinNumElements: RISCV::RVVBitsPerBlock /
1313 VecTy.getElementType().getSizeInBits(),
1314 ScalarTy: VecTy.getElementType());
1315}
1316
1317bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
1318 MachineIRBuilder &MIB) const {
1319 GExtractSubvector &ES = cast<GExtractSubvector>(Val&: MI);
1320
1321 MachineRegisterInfo &MRI = *MIB.getMRI();
1322
1323 Register Dst = ES.getReg(Idx: 0);
1324 Register Src = ES.getSrcVec();
1325 uint64_t Idx = ES.getIndexImm();
1326
1327 // With an index of 0 this is a cast-like subvector, which can be performed
1328 // with subregister operations.
1329 if (Idx == 0)
1330 return true;
1331
1332 LLT LitTy = MRI.getType(Reg: Dst);
1333 LLT BigTy = MRI.getType(Reg: Src);
1334
1335 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1336 // We can't slide this mask vector up indexed by its i1 elements.
1337 // This poses a problem when we wish to insert a scalable vector which
1338 // can't be re-expressed as a larger type. Just choose the slow path and
1339 // extend to a larger type, then truncate back down.
1340 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1341 LLT ExtLitTy = LitTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1342 auto BigZExt = MIB.buildZExt(Res: ExtBigTy, Op: Src);
1343 auto ExtractZExt = MIB.buildExtractSubvector(Res: ExtLitTy, Src: BigZExt, Index: Idx);
1344 auto SplatZero = MIB.buildSplatVector(
1345 Res: ExtLitTy, Val: MIB.buildConstant(Res: ExtLitTy.getElementType(), Val: 0));
1346 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: ExtractZExt, Op1: SplatZero);
1347 MI.eraseFromParent();
1348 return true;
1349 }
1350
1351 // extract_subvector scales the index by vscale if the subvector is scalable,
1352 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
1353 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1354 MVT LitTyMVT = getMVTForLLT(Ty: LitTy);
1355 auto Decompose =
1356 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1357 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: LitTyMVT, InsertExtractIdx: Idx, TRI);
1358 unsigned RemIdx = Decompose.second;
1359
1360 // If the Idx has been completely eliminated then this is a subvector extract
1361 // which naturally aligns to a vector register. These can easily be handled
1362 // using subregister manipulation.
1363 if (RemIdx == 0)
1364 return true;
1365
1366 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
1367 // was > M1 then the index would need to be a multiple of VLMAX, and so would
1368 // divide exactly.
1369 assert(
1370 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||
1371 RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVVType::LMUL_1);
1372
1373 // If the vector type is an LMUL-group type, extract a subvector equal to the
1374 // nearest full vector register type.
1375 LLT InterLitTy = BigTy;
1376 Register Vec = Src;
1377 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1378 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1379 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
1380 // we should have successfully decomposed the extract into a subregister.
1381 assert(Decompose.first != RISCV::NoSubRegister);
1382 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1383 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
1384 // specified on the source Register (the equivalent) since generic virtual
1385 // register does not allow subregister index.
1386 Vec = MIB.buildExtractSubvector(Res: InterLitTy, Src, Index: Idx - RemIdx).getReg(Idx: 0);
1387 }
1388
1389 // Slide this vector register down by the desired number of elements in order
1390 // to place the desired subvector starting at element 0.
1391 const LLT XLenTy(STI.getXLenVT());
1392 auto SlidedownAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1393 auto [Mask, VL] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1394 uint64_t Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
1395 auto Slidedown = MIB.buildInstr(
1396 Opc: RISCV::G_VSLIDEDOWN_VL, DstOps: {InterLitTy},
1397 SrcOps: {MIB.buildUndef(Res: InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
1398
1399 // Now the vector is in the right position, extract our final subvector. This
1400 // should resolve to a COPY.
1401 MIB.buildExtractSubvector(Res: Dst, Src: Slidedown, Index: 0);
1402
1403 MI.eraseFromParent();
1404 return true;
1405}
1406
1407bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,
1408 LegalizerHelper &Helper,
1409 MachineIRBuilder &MIB) const {
1410 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
1411
1412 MachineRegisterInfo &MRI = *MIB.getMRI();
1413
1414 Register Dst = IS.getReg(Idx: 0);
1415 Register BigVec = IS.getBigVec();
1416 Register LitVec = IS.getSubVec();
1417 uint64_t Idx = IS.getIndexImm();
1418
1419 LLT BigTy = MRI.getType(Reg: BigVec);
1420 LLT LitTy = MRI.getType(Reg: LitVec);
1421
1422 if (Idx == 0 && mi_match(R: BigVec, MRI, P: m_GImplicitDef()))
1423 return true;
1424
1425 // We don't have the ability to slide mask vectors up indexed by their i1
1426 // elements; the smallest we can do is i8. Often we are able to bitcast to
1427 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i8
1428 // vectors and truncate down after the insert.
1429 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1430 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();
1431 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();
1432 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)
1433 return Helper.bitcast(
1434 MI&: IS, TypeIdx: 0,
1435 Ty: LLT::vector(EC: BigTy.getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8));
1436
1437 // We can't slide this mask vector up indexed by its i1 elements.
1438 // This poses a problem when we wish to insert a scalable vector which
1439 // can't be re-expressed as a larger type. Just choose the slow path and
1440 // extend to a larger type, then truncate back down.
1441 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1442 return Helper.widenScalar(MI&: IS, TypeIdx: 0, WideTy: ExtBigTy);
1443 }
1444
1445 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1446 unsigned SubRegIdx, RemIdx;
1447 std::tie(args&: SubRegIdx, args&: RemIdx) =
1448 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1449 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: getMVTForLLT(Ty: LitTy), InsertExtractIdx: Idx, TRI);
1450
1451 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
1452 assert(isPowerOf2_64(
1453 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));
1454 bool ExactlyVecRegSized =
1455 STI.expandVScale(X: LitTy.getSizeInBits())
1456 .isKnownMultipleOf(RHS: STI.expandVScale(X: VecRegSize));
1457
1458 // If the Idx has been completely eliminated and this subvector's size is a
1459 // vector register or a multiple thereof, or the surrounding elements are
1460 // undef, then this is a subvector insert which naturally aligns to a vector
1461 // register. These can easily be handled using subregister manipulation.
1462 if (RemIdx == 0 && ExactlyVecRegSized)
1463 return true;
1464
1465 // If the subvector is smaller than a vector register, then the insertion
1466 // must preserve the undisturbed elements of the register. We do this by
1467 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
1468 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
1469 // subvector within the vector register, and an INSERT_SUBVECTOR of that
1470 // LMUL=1 type back into the larger vector (resolving to another subregister
1471 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
1472 // to avoid allocating a large register group to hold our subvector.
1473
1474 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
1475 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
1476 // (in our case undisturbed). This means we can set up a subvector insertion
1477 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
1478 // size of the subvector.
1479 const LLT XLenTy(STI.getXLenVT());
1480 LLT InterLitTy = BigTy;
1481 Register AlignedExtract = BigVec;
1482 unsigned AlignedIdx = Idx - RemIdx;
1483 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1484 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1485 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1486 // Extract a subvector equal to the nearest full vector register type. This
1487 // should resolve to a G_EXTRACT on a subreg.
1488 AlignedExtract =
1489 MIB.buildExtractSubvector(Res: InterLitTy, Src: BigVec, Index: AlignedIdx).getReg(Idx: 0);
1490 }
1491
1492 auto Insert = MIB.buildInsertSubvector(Res: InterLitTy, Src0: MIB.buildUndef(Res: InterLitTy),
1493 Src1: LitVec, Index: 0);
1494
1495 auto [Mask, _] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1496 auto VL = MIB.buildVScale(Res: XLenTy, MinElts: LitTy.getElementCount().getKnownMinValue());
1497
1498 // If we're inserting into the lowest elements, use a tail undisturbed
1499 // vmv.v.v.
1500 MachineInstrBuilder Inserted;
1501 bool NeedInsertSubvec =
1502 TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(), RHS: InterLitTy.getSizeInBits());
1503 Register InsertedDst =
1504 NeedInsertSubvec ? MRI.createGenericVirtualRegister(Ty: InterLitTy) : Dst;
1505 if (RemIdx == 0) {
1506 Inserted = MIB.buildInstr(Opc: RISCV::G_VMV_V_V_VL, DstOps: {InsertedDst},
1507 SrcOps: {AlignedExtract, Insert, VL});
1508 } else {
1509 auto SlideupAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1510 // Construct the vector length corresponding to RemIdx + length(LitTy).
1511 VL = MIB.buildAdd(Dst: XLenTy, Src0: SlideupAmt, Src1: VL);
1512 // Use tail agnostic policy if we're inserting over InterLitTy's tail.
1513 ElementCount EndIndex =
1514 ElementCount::getScalable(MinVal: RemIdx) + LitTy.getElementCount();
1515 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
1516 if (STI.expandVScale(X: EndIndex) ==
1517 STI.expandVScale(X: InterLitTy.getElementCount()))
1518 Policy = RISCVVType::TAIL_AGNOSTIC;
1519
1520 Inserted =
1521 MIB.buildInstr(Opc: RISCV::G_VSLIDEUP_VL, DstOps: {InsertedDst},
1522 SrcOps: {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});
1523 }
1524
1525 // If required, insert this subvector back into the correct vector register.
1526 // This should resolve to an INSERT_SUBREG instruction.
1527 if (NeedInsertSubvec)
1528 MIB.buildInsertSubvector(Res: Dst, Src0: BigVec, Src1: Inserted, Index: AlignedIdx);
1529
1530 MI.eraseFromParent();
1531 return true;
1532}
1533
1534bool RISCVLegalizerInfo::legalizeBitreverse(MachineInstr &MI,
1535 MachineIRBuilder &MIB) const {
1536 assert(MI.getOpcode() == TargetOpcode::G_BITREVERSE && "Unexpected opcode");
1537
1538 if (!STI.hasStdExtZbkb())
1539 return false;
1540
1541 MachineRegisterInfo &MRI = *MIB.getMRI();
1542
1543 Register Dst = MI.getOperand(i: 0).getReg();
1544 Register Src = MI.getOperand(i: 1).getReg();
1545
1546 if (!MRI.getType(Reg: Dst).isScalar(Size: 8))
1547 return false;
1548
1549 auto WideSrc = MIB.buildAnyExt(Res: sXLen, Op: Src);
1550 auto Brev = MIB.buildInstr(Opc: RISCV::G_BREV8, DstOps: {sXLen}, SrcOps: {WideSrc.getReg(Idx: 0)});
1551 MIB.buildTrunc(Res: Dst, Op: Brev.getReg(Idx: 0));
1552
1553 MI.eraseFromParent();
1554 return true;
1555}
1556
1557static unsigned getRISCVWOpcode(unsigned Opcode) {
1558 switch (Opcode) {
1559 default:
1560 llvm_unreachable("Unexpected opcode");
1561 case TargetOpcode::G_ASHR:
1562 return RISCV::G_SRAW;
1563 case TargetOpcode::G_LSHR:
1564 return RISCV::G_SRLW;
1565 case TargetOpcode::G_SHL:
1566 return RISCV::G_SLLW;
1567 case TargetOpcode::G_SDIV:
1568 return RISCV::G_DIVW;
1569 case TargetOpcode::G_UDIV:
1570 return RISCV::G_DIVUW;
1571 case TargetOpcode::G_UREM:
1572 return RISCV::G_REMUW;
1573 case TargetOpcode::G_ROTL:
1574 return RISCV::G_ROLW;
1575 case TargetOpcode::G_ROTR:
1576 return RISCV::G_RORW;
1577 case TargetOpcode::G_CTLZ:
1578 return RISCV::G_CLZW;
1579 case TargetOpcode::G_CTTZ:
1580 return RISCV::G_CTZW;
1581 case TargetOpcode::G_CTLS:
1582 return RISCV::G_CLSW;
1583 case TargetOpcode::G_FPTOSI:
1584 return RISCV::G_FCVT_W_RV64;
1585 case TargetOpcode::G_FPTOUI:
1586 return RISCV::G_FCVT_WU_RV64;
1587 }
1588}
1589
1590bool RISCVLegalizerInfo::legalizeCustom(
1591 LegalizerHelper &Helper, MachineInstr &MI,
1592 LostDebugLocObserver &LocObserver) const {
1593 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1594 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1595 MachineFunction &MF = *MI.getParent()->getParent();
1596 switch (MI.getOpcode()) {
1597 default:
1598 // No idea what to do.
1599 return false;
1600 case TargetOpcode::G_ABS:
1601 return Helper.lowerAbsToMaxNeg(MI);
1602 case TargetOpcode::G_FCONSTANT: {
1603 const APFloat &FVal = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1604
1605 // Convert G_FCONSTANT to G_CONSTANT.
1606 Register DstReg = MI.getOperand(i: 0).getReg();
1607 MIRBuilder.buildConstant(Res: DstReg, Val: FVal.bitcastToAPInt());
1608
1609 MI.eraseFromParent();
1610 return true;
1611 }
1612 case TargetOpcode::G_CONSTANT: {
1613 const Function &F = MF.getFunction();
1614 // TODO: if PSI and BFI are present, add " ||
1615 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".
1616 bool ShouldOptForSize = F.hasOptSize();
1617 const ConstantInt *ConstVal = MI.getOperand(i: 1).getCImm();
1618 if (!shouldBeInConstantPool(APImm: ConstVal->getValue(), ShouldOptForSize))
1619 return true;
1620 return Helper.lowerConstant(MI);
1621 }
1622 case TargetOpcode::G_SUB:
1623 case TargetOpcode::G_ADD: {
1624 Helper.Observer.changingInstr(MI);
1625 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1626 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1627
1628 Register DstALU = MRI.createGenericVirtualRegister(Ty: sXLen);
1629
1630 MachineOperand &MO = MI.getOperand(i: 0);
1631 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
1632 auto DstSext = MIRBuilder.buildSExtInReg(Res: sXLen, Op: DstALU, ImmOp: 32);
1633
1634 MIRBuilder.buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: {MO}, SrcOps: {DstSext});
1635 MO.setReg(DstALU);
1636
1637 Helper.Observer.changedInstr(MI);
1638 return true;
1639 }
1640 case TargetOpcode::G_ASHR:
1641 case TargetOpcode::G_LSHR:
1642 case TargetOpcode::G_SHL: {
1643 if (getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI)) {
1644 // We don't need a custom node for shift by constant. Just widen the
1645 // source and the shift amount.
1646 unsigned ExtOpc = TargetOpcode::G_ANYEXT;
1647 if (MI.getOpcode() == TargetOpcode::G_ASHR)
1648 ExtOpc = TargetOpcode::G_SEXT;
1649 else if (MI.getOpcode() == TargetOpcode::G_LSHR)
1650 ExtOpc = TargetOpcode::G_ZEXT;
1651
1652 Helper.Observer.changingInstr(MI);
1653 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: ExtOpc);
1654 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
1655 Helper.widenScalarDst(MI, WideTy: sXLen);
1656 Helper.Observer.changedInstr(MI);
1657 return true;
1658 }
1659
1660 Helper.Observer.changingInstr(MI);
1661 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1662 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1663 Helper.widenScalarDst(MI, WideTy: sXLen);
1664 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1665 Helper.Observer.changedInstr(MI);
1666 return true;
1667 }
1668 case TargetOpcode::G_SDIV:
1669 case TargetOpcode::G_UDIV:
1670 case TargetOpcode::G_UREM:
1671 case TargetOpcode::G_ROTL:
1672 case TargetOpcode::G_ROTR: {
1673 Helper.Observer.changingInstr(MI);
1674 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1675 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1676 Helper.widenScalarDst(MI, WideTy: sXLen);
1677 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1678 Helper.Observer.changedInstr(MI);
1679 return true;
1680 }
1681 case TargetOpcode::G_CTLZ:
1682 case TargetOpcode::G_CTTZ:
1683 case TargetOpcode::G_CTLS: {
1684 Helper.Observer.changingInstr(MI);
1685 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1686 Helper.widenScalarDst(MI, WideTy: sXLen);
1687 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1688 Helper.Observer.changedInstr(MI);
1689 return true;
1690 }
1691 case TargetOpcode::G_FPTOSI:
1692 case TargetOpcode::G_FPTOUI: {
1693 Helper.Observer.changingInstr(MI);
1694 Helper.widenScalarDst(MI, WideTy: sXLen);
1695 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1696 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RTZ));
1697 Helper.Observer.changedInstr(MI);
1698 return true;
1699 }
1700 case TargetOpcode::G_LROUND: {
1701 // The (i32 any_lround) Pat is IsRV32-only; on RV64 lower to
1702 // riscv_fcvt_w_rv64 with FRM_RMM.
1703 Helper.Observer.changingInstr(MI);
1704 Helper.widenScalarDst(MI, WideTy: sXLen);
1705 MI.setDesc(MIRBuilder.getTII().get(Opcode: RISCV::G_FCVT_W_RV64));
1706 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RMM));
1707 Helper.Observer.changedInstr(MI);
1708 return true;
1709 }
1710 case TargetOpcode::G_READCYCLECOUNTER:
1711 case TargetOpcode::G_READSTEADYCOUNTER:
1712 return legalizeReadCounter(MI, MIRBuilder, Observer&: Helper.Observer);
1713 case TargetOpcode::G_IS_FPCLASS: {
1714 Register GISFPCLASS = MI.getOperand(i: 0).getReg();
1715 Register Src = MI.getOperand(i: 1).getReg();
1716 const MachineOperand &ImmOp = MI.getOperand(i: 2);
1717 MachineIRBuilder MIB(MI);
1718
1719 // Turn LLVM IR's floating point classes to that in RISC-V,
1720 // by simply rotating the 10-bit immediate right by two bits.
1721 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));
1722 auto FClassMask = MIB.buildConstant(Res: sXLen, Val: GFpClassImm.rotr(rotateAmt: 2).zext(width: XLen));
1723 auto ConstZero = MIB.buildConstant(Res: sXLen, Val: 0);
1724
1725 auto GFClass = MIB.buildInstr(Opc: RISCV::G_FCLASS, DstOps: {sXLen}, SrcOps: {Src});
1726 auto And = MIB.buildAnd(Dst: sXLen, Src0: GFClass, Src1: FClassMask);
1727 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: GISFPCLASS, Op0: And, Op1: ConstZero);
1728
1729 MI.eraseFromParent();
1730 return true;
1731 }
1732 case TargetOpcode::G_BRJT:
1733 return legalizeBRJT(MI, MIRBuilder);
1734 case TargetOpcode::G_VASTART:
1735 return legalizeVAStart(MI, MIRBuilder);
1736 case TargetOpcode::G_VSCALE:
1737 return legalizeVScale(MI, MIB&: MIRBuilder);
1738 case TargetOpcode::G_ZEXT:
1739 case TargetOpcode::G_SEXT:
1740 case TargetOpcode::G_ANYEXT:
1741 return legalizeExt(MI, MIB&: MIRBuilder);
1742 case TargetOpcode::G_SPLAT_VECTOR:
1743 return legalizeSplatVector(MI, MIB&: MIRBuilder);
1744 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1745 return legalizeExtractSubvector(MI, MIB&: MIRBuilder);
1746 case TargetOpcode::G_INSERT_SUBVECTOR:
1747 return legalizeInsertSubvector(MI, Helper, MIB&: MIRBuilder);
1748 case TargetOpcode::G_BITREVERSE:
1749 return legalizeBitreverse(MI, MIB&: MIRBuilder);
1750 case TargetOpcode::G_LOAD:
1751 case TargetOpcode::G_STORE:
1752 return legalizeLoadStore(MI, Helper, MIB&: MIRBuilder);
1753 }
1754
1755 llvm_unreachable("expected switch to return");
1756}
1757