1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for RISC-V.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "RISCVLegalizerInfo.h"
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVSubtarget.h"
17#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/MachineConstantPool.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/TargetOpcodes.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsRISCV.h"
32#include "llvm/IR/Type.h"
33
34using namespace llvm;
35using namespace LegalityPredicates;
36using namespace LegalizeMutations;
37using namespace MIPatternMatch;
38
39static LegalityPredicate
40typeIsLegalIntOrFPVec(unsigned TypeIdx,
41 std::initializer_list<LLT> IntOrFPVecTys,
42 const RISCVSubtarget &ST) {
43 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
44 return ST.hasVInstructions() &&
45 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
46 ST.hasVInstructionsI64()) &&
47 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
48 ST.getELen() == 64);
49 };
50
51 return all(P0: typeInSet(TypeIdx, TypesInit: IntOrFPVecTys), P1: P);
52}
53
54static LegalityPredicate
55typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
56 const RISCVSubtarget &ST) {
57 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
58 return ST.hasVInstructions() &&
59 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
60 ST.getELen() == 64);
61 };
62 return all(P0: typeInSet(TypeIdx, TypesInit: BoolVecTys), P1: P);
63}
64
65static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
66 std::initializer_list<LLT> PtrVecTys,
67 const RISCVSubtarget &ST) {
68 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
69 return ST.hasVInstructions() &&
70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
71 ST.getELen() == 64) &&
72 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
73 Query.Types[TypeIdx].getScalarSizeInBits() == 32);
74 };
75 return all(P0: typeInSet(TypeIdx, TypesInit: PtrVecTys), P1: P);
76}
77
78RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
79 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(SizeInBits: XLen)) {
80 const LLT sDoubleXLen = LLT::scalar(SizeInBits: 2 * XLen);
81 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: XLen);
82 const LLT s1 = LLT::scalar(SizeInBits: 1);
83 const LLT s8 = LLT::scalar(SizeInBits: 8);
84 const LLT s16 = LLT::scalar(SizeInBits: 16);
85 const LLT f16 = LLT::float16();
86 const LLT s32 = LLT::scalar(SizeInBits: 32);
87 const LLT s64 = LLT::scalar(SizeInBits: 64);
88 const LLT s128 = LLT::scalar(SizeInBits: 128);
89
90 const LLT nxv1s1 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s1);
91 const LLT nxv2s1 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s1);
92 const LLT nxv4s1 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s1);
93 const LLT nxv8s1 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s1);
94 const LLT nxv16s1 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s1);
95 const LLT nxv32s1 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s1);
96 const LLT nxv64s1 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s1);
97
98 const LLT nxv1s8 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s8);
99 const LLT nxv2s8 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s8);
100 const LLT nxv4s8 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s8);
101 const LLT nxv8s8 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s8);
102 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
103 const LLT nxv32s8 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s8);
104 const LLT nxv64s8 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s8);
105
106 const LLT nxv1s16 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s16);
107 const LLT nxv2s16 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s16);
108 const LLT nxv4s16 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s16);
109 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
110 const LLT nxv16s16 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s16);
111 const LLT nxv32s16 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s16);
112
113 const LLT nxv1s32 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s32);
114 const LLT nxv2s32 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s32);
115 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
116 const LLT nxv8s32 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s32);
117 const LLT nxv16s32 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s32);
118
119 const LLT nxv1s64 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s64);
120 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
121 const LLT nxv4s64 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s64);
122 const LLT nxv8s64 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s64);
123
124 const LLT nxv1p0 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: p0);
125 const LLT nxv2p0 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: p0);
126 const LLT nxv4p0 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: p0);
127 const LLT nxv8p0 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: p0);
128 const LLT nxv16p0 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: p0);
129
130 using namespace TargetOpcode;
131
132 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
133
134 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,
135 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
136 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
137 nxv1s64, nxv2s64, nxv4s64, nxv8s64};
138
139 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
140
141 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
142 .legalFor(Types: {sXLen})
143 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
144 .customFor(Pred: ST.is64Bit(), Types: {s32})
145 .widenScalarToNextPow2(TypeIdx: 0)
146 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
147
148 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
149 .legalFor(Types: {sXLen})
150 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
151 .widenScalarToNextPow2(TypeIdx: 0)
152 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
153
154 getActionDefinitionsBuilder(
155 Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO, G_READ_REGISTER, G_WRITE_REGISTER})
156 .lower();
157
158 getActionDefinitionsBuilder(Opcodes: {G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})
159 .minScalar(TypeIdx: 0, Ty: sXLen)
160 .lower();
161
162 // TODO: Use Vector Single-Width Saturating Instructions for vector types.
163 getActionDefinitionsBuilder(
164 Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})
165 .lower();
166
167 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
168 .legalFor(Types: {{sXLen, sXLen}})
169 .customFor(Pred: ST.is64Bit(), Types: {{s32, s32}})
170 .widenScalarToNextPow2(TypeIdx: 0)
171 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
172 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
173
174 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
175 .legalFor(Types: {{s32, s16}})
176 .legalFor(Pred: ST.is64Bit(), Types: {{s64, s16}, {s64, s32}})
177 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
178 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
179 .customIf(Predicate: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))
180 .maxScalar(TypeIdx: 0, Ty: sXLen);
181
182 getActionDefinitionsBuilder(Opcode: G_TRUNC).alwaysLegal();
183
184 {
185 LegalityPredicate ValidSextInRegWidth = all(P0: sizeIs(TypeIdx: 0, Size: 64), P1: immIs(ImmIdx: 0, Imm: 32));
186
187 if (STI.hasStdExtZbb())
188 ValidSextInRegWidth =
189 LegalityPredicates::any(P0: ValidSextInRegWidth, P1: immInSet(ImmIdx: 0, ImmsInit: {8, 16}));
190
191 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
192 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: sXLen), P1: ValidSextInRegWidth))
193 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
194 .lower();
195 }
196
197 // Merge/Unmerge
198 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
199 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Opcode: Op);
200 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
201 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
202 if (XLen == 32 && ST.hasStdExtD()) {
203 MergeUnmergeActions.legalIf(
204 Predicate: all(P0: typeIs(TypeIdx: BigTyIdx, TypesInit: s64), P1: typeIs(TypeIdx: LitTyIdx, TypesInit: s32)));
205 }
206 MergeUnmergeActions.widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: XLen)
207 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: XLen)
208 .clampScalar(TypeIdx: LitTyIdx, MinTy: sXLen, MaxTy: sXLen)
209 .clampScalar(TypeIdx: BigTyIdx, MinTy: sXLen, MaxTy: sXLen);
210 }
211
212 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR}).lower();
213
214 getActionDefinitionsBuilder(Opcodes: {G_ROTR, G_ROTL})
215 .legalFor(Pred: ST.hasStdExtZbb() || ST.hasStdExtZbkb(), Types: {{sXLen, sXLen}})
216 .customFor(Pred: ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),
217 Types: {{s32, s32}})
218 .lower();
219
220 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
221 .customFor(Pred: ST.hasStdExtZbkb(), Types: {s8})
222 .maxScalar(TypeIdx: 0, Ty: sXLen)
223 .lower();
224
225 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
226 Predicate: all(P0: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
227 P1: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST)),
228 P1: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST),
229 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))));
230
231 auto &BSWAPActions = getActionDefinitionsBuilder(Opcode: G_BSWAP);
232 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
233 BSWAPActions.legalFor(Types: {sXLen}).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
234 else
235 BSWAPActions.maxScalar(TypeIdx: 0, Ty: sXLen).lower();
236
237 getActionDefinitionsBuilder(Opcode: G_CLMUL)
238 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
239 .unsupported();
240
241 getActionDefinitionsBuilder(Opcode: G_CLMULH)
242 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
243 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbkc(), Types: {s32})
244 .unsupported();
245
246 // CLMULR is Zbc-only; Zbkc is a subset that has CLMUL/CLMULH but not CLMULR.
247 getActionDefinitionsBuilder(Opcode: G_CLMULR)
248 .legalFor(Pred: ST.hasStdExtZbc(), Types: {sXLen})
249 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbc(), Types: {s32})
250 .unsupported();
251
252 auto &CountZerosActions = getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTTZ});
253 auto &CountZerosPoisonActions =
254 getActionDefinitionsBuilder(Opcodes: {G_CTLZ_ZERO_POISON, G_CTTZ_ZERO_POISON});
255 if (ST.hasStdExtZbb()) {
256 CountZerosActions.legalFor(Types: {{sXLen, sXLen}})
257 .customFor(Types: {{s32, s32}})
258 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
259 .widenScalarToNextPow2(TypeIdx: 0)
260 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
261 } else {
262 CountZerosActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
263 CountZerosPoisonActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
264 }
265 CountZerosPoisonActions.lower();
266
267 auto &CountSignActions = getActionDefinitionsBuilder(Opcode: G_CTLS);
268 if (ST.hasStdExtP()) {
269 CountSignActions.legalFor(Types: {{sXLen, sXLen}})
270 .customFor(Types: {{s32, s32}})
271 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
272 .widenScalarToNextPow2(TypeIdx: 0)
273 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
274 } else {
275 CountSignActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
276 }
277
278 auto &CTPOPActions = getActionDefinitionsBuilder(Opcode: G_CTPOP);
279 if (ST.hasStdExtZbb()) {
280 CTPOPActions.legalFor(Types: {{sXLen, sXLen}})
281 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
282 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
283 } else {
284 CTPOPActions.widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8)
285 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sXLen)
286 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0)
287 .lower();
288 }
289
290 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
291 .legalFor(Types: {p0})
292 .legalFor(Pred: !ST.is64Bit(), Types: {s32})
293 .customFor(Pred: ST.is64Bit(), Types: {s64})
294 .widenScalarToNextPow2(TypeIdx: 0)
295 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
296
297 // TODO: transform illegal vector types into legal vector type
298 getActionDefinitionsBuilder(Opcode: G_FREEZE)
299 .legalFor(Types: {s16, s32, p0})
300 .legalFor(Pred: ST.is64Bit(), Types: {s64})
301 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
302 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
303 .widenScalarToNextPow2(TypeIdx: 0)
304 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen);
305
306 // TODO: transform illegal vector types into legal vector type
307 // TODO: Merge with G_FREEZE?
308 getActionDefinitionsBuilder(
309 Opcodes: {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})
310 .legalFor(Types: {s32, sXLen, p0})
311 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
312 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
313 .widenScalarToNextPow2(TypeIdx: 0)
314 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
315
316 getActionDefinitionsBuilder(Opcode: G_ICMP)
317 .legalFor(Types: {{sXLen, sXLen}, {sXLen, p0}})
318 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
319 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
320 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, MinSize: 8)
321 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
322 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
323
324 getActionDefinitionsBuilder(Opcode: G_SELECT)
325 .legalFor(Types: {{s32, sXLen}, {p0, sXLen}})
326 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
327 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
328 .legalFor(Pred: XLen == 64 || ST.hasStdExtD(), Types: {{s64, sXLen}})
329 .widenScalarToNextPow2(TypeIdx: 0)
330 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)
331 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
332
333 auto &LoadActions = getActionDefinitionsBuilder(Opcode: G_LOAD);
334 auto &StoreActions = getActionDefinitionsBuilder(Opcode: G_STORE);
335 auto &ExtLoadActions = getActionDefinitionsBuilder(Opcodes: {G_SEXTLOAD, G_ZEXTLOAD});
336
337 // Return the alignment needed for scalar memory ops. If unaligned scalar mem
338 // is supported, we only require byte alignment. Otherwise, we need the memory
339 // op to be natively aligned.
340 auto getScalarMemAlign = [&ST](unsigned Size) {
341 return ST.enableUnalignedScalarMem() ? 8 : Size;
342 };
343
344 LoadActions.legalForTypesWithMemDesc(
345 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
346 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
347 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
348 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
349 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
350 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
351 StoreActions.legalForTypesWithMemDesc(
352 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
353 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
354 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
355 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
356 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
357 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
358 ExtLoadActions.legalForTypesWithMemDesc(
359 TypesAndMemDesc: {{.Type0: sXLen, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
360 {.Type0: sXLen, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)}});
361 if (XLen == 64) {
362 LoadActions.legalForTypesWithMemDesc(
363 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
364 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
365 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
366 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
367 StoreActions.legalForTypesWithMemDesc(
368 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
369 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
370 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
371 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
372 ExtLoadActions.legalForTypesWithMemDesc(
373 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)}});
374 } else if (ST.hasStdExtD()) {
375 LoadActions.legalForTypesWithMemDesc(
376 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
377 StoreActions.legalForTypesWithMemDesc(
378 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
379 }
380
381 // Vector loads/stores.
382 if (ST.hasVInstructions()) {
383 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
384 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
385 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
386 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
387 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
388 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
389 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
390 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
391 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
392 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
393 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
394 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
395 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
396 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
397 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
398 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
399 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
400 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
401 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
402 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
403 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
404 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
405 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
406 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
407 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
408 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
409 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
410 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
411 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
412 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
413
414 if (ST.getELen() == 64) {
415 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
416 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
417 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
418 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
419 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
420 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
421 }
422
423 if (ST.hasVInstructionsI64()) {
424 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
425 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
426 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
427 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
428 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
429 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
430 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
431 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
432 }
433
434 // we will take the custom lowering logic if we have scalable vector types
435 // with non-standard alignments
436 LoadActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
437 StoreActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
438
439 // Pointers require that XLen sized elements are legal.
440 if (XLen <= ST.getELen()) {
441 LoadActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
442 StoreActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
443 }
444 }
445
446 LoadActions.widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
447 .lowerIfMemSizeNotByteSizePow2()
448 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
449 .lower();
450 StoreActions
451 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
452 .lowerIfMemSizeNotByteSizePow2()
453 .lower();
454
455 ExtLoadActions.widenScalarToNextPow2(TypeIdx: 0).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).lower();
456
457 getActionDefinitionsBuilder(Opcodes: {G_PTR_ADD, G_PTRMASK}).legalFor(Types: {{p0, sXLen}});
458
459 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
460 .legalFor(Types: {{sXLen, p0}})
461 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
462
463 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
464 .legalFor(Types: {{p0, sXLen}})
465 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
466
467 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
468
469 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {sXLen}).minScalar(TypeIdx: 0, Ty: sXLen);
470
471 getActionDefinitionsBuilder(Opcode: G_BRJT).customFor(Types: {{p0, sXLen}});
472
473 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
474
475 getActionDefinitionsBuilder(Opcode: G_PHI)
476 .legalFor(Types: {p0, s32, sXLen})
477 .widenScalarToNextPow2(TypeIdx: 0)
478 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
479
480 getActionDefinitionsBuilder(Opcodes: {G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
481 .legalFor(Types: {p0});
482
483 if (ST.hasStdExtZmmul()) {
484 getActionDefinitionsBuilder(Opcode: G_MUL)
485 .legalFor(Types: {sXLen})
486 .widenScalarToNextPow2(TypeIdx: 0)
487 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
488
489 // clang-format off
490 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
491 .legalFor(Types: {sXLen})
492 .lower();
493 // clang-format on
494
495 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO}).minScalar(TypeIdx: 0, Ty: sXLen).lower();
496 } else {
497 getActionDefinitionsBuilder(Opcode: G_MUL)
498 .libcallFor(Types: {sXLen, sDoubleXLen})
499 .widenScalarToNextPow2(TypeIdx: 0)
500 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen);
501
502 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).lowerFor(Types: {sXLen});
503
504 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
505 .minScalar(TypeIdx: 0, Ty: sXLen)
506 // Widen sXLen to sDoubleXLen so we can use a single libcall to get
507 // the low bits for the mul result and high bits to do the overflow
508 // check.
509 .widenScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: sXLen),
510 Mutation: LegalizeMutations::changeTo(TypeIdx: 0, Ty: sDoubleXLen))
511 .lower();
512 }
513
514 if (ST.hasStdExtM()) {
515 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV, G_UREM})
516 .legalFor(Types: {sXLen})
517 .customFor(Types: {s32})
518 .libcallFor(Types: {sDoubleXLen})
519 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sDoubleXLen)
520 .widenScalarToNextPow2(TypeIdx: 0);
521 getActionDefinitionsBuilder(Opcode: G_SREM)
522 .legalFor(Types: {sXLen})
523 .libcallFor(Types: {sDoubleXLen})
524 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
525 .widenScalarToNextPow2(TypeIdx: 0);
526 } else {
527 getActionDefinitionsBuilder(Opcodes: {G_UDIV, G_SDIV, G_UREM, G_SREM})
528 .libcallFor(Types: {sXLen, sDoubleXLen})
529 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
530 .widenScalarToNextPow2(TypeIdx: 0);
531 }
532
533 // TODO: Use libcall for sDoubleXLen.
534 getActionDefinitionsBuilder(Opcodes: {G_SDIVREM, G_UDIVREM}).lower();
535
536 getActionDefinitionsBuilder(Opcode: G_ABS)
537 .customFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
538 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
539 .lower();
540
541 getActionDefinitionsBuilder(Opcodes: {G_ABDS, G_ABDU})
542 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
543 .lower();
544
545 getActionDefinitionsBuilder(Opcodes: {G_UMAX, G_UMIN, G_SMAX, G_SMIN})
546 .legalFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
547 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
548 .lower();
549
550 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
551
552 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
553
554 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
555
556 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
557
558 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
559 .lower();
560
561 // On RV64 the 64-bit counter CSRs (cycle/time) are read directly. On RV32
562 // they are custom-legally lowered to a re-read-the-high-half loop (see
563 // legalizeReadCounter).
564 getActionDefinitionsBuilder(Opcodes: {G_READCYCLECOUNTER, G_READSTEADYCOUNTER})
565 .legalFor(Pred: ST.is64Bit(), Types: {s64})
566 .customFor(Pred: !ST.is64Bit(), Types: {s64});
567
568 // FP Operations
569
570 // FIXME: Support s128 for rv32 when libcall handling is able to use sret.
571 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,
572 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,
573 G_FMINIMUMNUM})
574 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
575 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
576 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
577 .libcallFor(Types: {s32, s64})
578 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
579
580 getActionDefinitionsBuilder(Opcodes: {G_FNEG, G_FABS})
581 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
582 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
583 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
584 .lowerFor(Types: {s32, s64, s128});
585
586 getActionDefinitionsBuilder(Opcode: G_FREM)
587 .libcallFor(Types: {s32, s64})
588 .libcallFor(Pred: ST.is64Bit(), Types: {s128})
589 .minScalar(TypeIdx: 0, Ty: s32)
590 .scalarize(TypeIdx: 0);
591
592 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
593 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, s32}})
594 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s64}, {s32, s64}, {s64, s32}})
595 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s16}, {s16, s32}, {s32, s16}})
596 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}, {s64, s16}})
597 .lower();
598
599 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
600 .legalFor(Pred: ST.hasStdExtD(), Types: {{s32, s64}})
601 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {{s16, s32}})
602 .legalFor(Pred: ST.hasStdExtZfhmin() && ST.hasStdExtD(), Types: {{s16, s64}})
603 .libcallFor(Types: {{s16, s32}, {s16, s64}, {s32, s64}})
604 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}});
605 getActionDefinitionsBuilder(Opcode: G_FPEXT)
606 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s32}})
607 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {{s32, s16}})
608 .legalFor(Pred: ST.hasStdExtZfhmin() && ST.hasStdExtD(), Types: {{s64, s16}})
609 .libcallFor(Pred: !ST.hasStdExtZfhmin(), Types: {{s32, s16}})
610 .libcallFor(Pred: !(ST.hasStdExtZfhmin() && ST.hasStdExtD()), Types: {{s64, s16}})
611 .libcallFor(Types: {{s64, s32}})
612 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}});
613
614 getActionDefinitionsBuilder(Opcode: G_FCMP)
615 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
616 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
617 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
618 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
619 .minScalar(TypeIdx: 1, Ty: s32)
620 .libcallFor(Types: {{sXLen, s32}, {sXLen, s64}})
621 .libcallFor(Pred: ST.is64Bit(), Types: {{sXLen, s128}});
622
623 // TODO: Support vector version of G_IS_FPCLASS.
624 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
625 .customFor(Pred: ST.hasStdExtF(), Types: {{s1, s32}})
626 .customFor(Pred: ST.hasStdExtD(), Types: {{s1, s64}})
627 .customFor(Pred: ST.hasStdExtZfh(), Types: {{s1, s16}})
628 .lower();
629
630 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
631 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
632 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
633 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {s16})
634 .customFor(Pred: !ST.is64Bit(), Types: {s32})
635 .customFor(Pred: ST.is64Bit(), Types: {s32, s64})
636 .lowerFor(Types: {s64, s128});
637
638 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
639 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
640 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
641 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
642 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
643 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
644 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
645 .widenScalarToNextPow2(TypeIdx: 0)
646 .minScalar(TypeIdx: 0, Ty: s32)
647 // The magnitude of a half is at most 65504, so with Zfh use fcvt.w[u].h
648 // and extend the i32 result. Otherwise promote the half source to float
649 // (via fcvt.s.h with Zfhmin, __extendhfsf2 without) and use the float
650 // conversion. On RV32, exclude i64 results here so that they get
651 // narrowed to i32 first.
652 .widenScalarIf(
653 Predicate: [=, &ST](const LegalityQuery &Query) {
654 return Query.Types[1] == f16 && !ST.hasStdExtZfh() &&
655 (ST.is64Bit() || Query.Types[0] == s32);
656 },
657 Mutation: changeTo(TypeIdx: 1, Ty: s32))
658 .libcallFor(Pred: !ST.hasStdExtZfhmin(), Types: {{s64, f16}})
659 .narrowScalarFor(Types: {{s64, f16}}, Mutation: changeTo(TypeIdx: 0, Ty: s32))
660 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
661 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}}) // FIXME RV32.
662 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}, {s128, s128}});
663
664 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
665 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
666 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
667 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
668 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
669 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
670 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
671 .minScalar(TypeIdx: 1, Ty: s32)
672 .libcallFor(Types: {{s32, s32},
673 {s64, s32},
674 {s32, s64},
675 {s64, s64},
676 {s32, s128},
677 {s64, s128}});
678
679 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
680 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
681 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
682 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
683 .minScalar(TypeIdx: 0, Ty: sXLen)
684 .minScalar(TypeIdx: 1, Ty: s32)
685 .libcallFor(Types: {{s32, s32},
686 {s64, s32},
687 {s32, s64},
688 {s64, s64},
689 {s32, s128},
690 {s64, s128}});
691
692 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
693 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, sXLen}})
694 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, sXLen}})
695 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, sXLen}})
696 .widenScalarToNextPow2(TypeIdx: 1)
697 // Promote to XLen if the operation is legal.
698 .widenScalarIf(
699 Predicate: [=, &ST](const LegalityQuery &Query) {
700 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
701 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
702 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
703 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
704 (ST.hasStdExtZfh() &&
705 Query.Types[0].getSizeInBits() == 16));
706 },
707 Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: sXLen))
708 // Otherwise only promote to s32 since we have si libcalls.
709 .minScalar(TypeIdx: 1, Ty: s32)
710 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
711 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}}) // FIXME RV32.
712 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}, {s128, s128}});
713
714 // FIXME: We can do custom inline expansion like SelectionDAG.
715 getActionDefinitionsBuilder(Opcodes: {G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
716 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
717 G_INTRINSIC_ROUNDEVEN})
718 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
719 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
720 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16})
721 .libcallFor(Types: {s32, s64})
722 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
723
724 getActionDefinitionsBuilder(Opcodes: {G_FMAXIMUM, G_FMINIMUM})
725 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
726 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
727 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16});
728
729 getActionDefinitionsBuilder(Opcodes: {G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
730 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
731 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
732 G_FTANH, G_FMODF, G_FSINCOS})
733 .libcallFor(Types: {s32, s64})
734 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
735 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
736 .libcallFor(Types: {{s32, s32}, {s64, s32}})
737 .libcallFor(Pred: ST.is64Bit(), Types: {s128, s32});
738
739 getActionDefinitionsBuilder(Opcode: G_FCANONICALIZE)
740 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
741 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
742 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16});
743
744 getActionDefinitionsBuilder(Opcode: G_VASTART).customFor(Types: {p0});
745
746 // va_list must be a pointer, but most sized types are pretty easy to handle
747 // as the destination.
748 getActionDefinitionsBuilder(Opcode: G_VAARG)
749 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types
750 // other than sXLen.
751 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
752 .lowerForCartesianProduct(Types0: {sXLen, p0}, Types1: {p0});
753
754 getActionDefinitionsBuilder(Opcode: G_VSCALE)
755 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
756 .customFor(Types: {sXLen});
757
758 auto &SplatActions =
759 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
760 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
761 P1: typeIs(TypeIdx: 1, TypesInit: sXLen)))
762 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: typeIs(TypeIdx: 1, TypesInit: s1)));
763 // Handle case of s64 element vectors on RV32. If the subtarget does not have
764 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget
765 // does have f64, then we don't know whether the type is an f64 or an i64,
766 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,
767 // depending on how the instructions it consumes are legalized. They are not
768 // legalized yet since legalization is in reverse postorder, so we cannot
769 // make the decision at this moment.
770 if (XLen == 32) {
771 if (ST.hasVInstructionsF64() && ST.hasStdExtD())
772 SplatActions.legalIf(Predicate: all(
773 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
774 else if (ST.hasVInstructionsI64())
775 SplatActions.customIf(Predicate: all(
776 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
777 }
778
779 SplatActions.clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
780
781 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
782 LLT DstTy = Query.Types[0];
783 LLT SrcTy = Query.Types[1];
784 return DstTy.getElementType() == LLT::scalar(SizeInBits: 1) &&
785 DstTy.getElementCount().getKnownMinValue() >= 8 &&
786 SrcTy.getElementCount().getKnownMinValue() >= 8;
787 };
788 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
789 // We don't have the ability to slide mask vectors down indexed by their
790 // i1 elements; the smallest we can do is i8. Often we are able to bitcast
791 // to equivalent i8 vectors.
792 .bitcastIf(
793 Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
794 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST), args: ExtractSubvecBitcastPred),
795 Mutation: [=](const LegalityQuery &Query) {
796 LLT CastTy = LLT::vector(
797 EC: Query.Types[0].getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8);
798 return std::pair(0, CastTy);
799 })
800 .customIf(Predicate: LegalityPredicates::any(
801 P0: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
802 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)),
803 P1: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
804 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST))));
805
806 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR)
807 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
808 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
809 .customIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
810 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)));
811
812 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
813 .lowerIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
814
815 getActionDefinitionsBuilder(Opcodes: {G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
816 G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
817 G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
818 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
819 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
820 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
821
822 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_SUB)
823 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
824 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
825 .lower();
826
827 getActionDefinitionsBuilder(
828 Opcodes: {G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
829 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
830 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
831 .unsupported();
832
833 getActionDefinitionsBuilder(Opcode: G_PREFETCH).legalIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p0));
834
835 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
836 LLT VecTy = Query.Types[0];
837 LLT EltTy = Query.Types[1];
838 return VecTy.getElementType() == EltTy;
839 };
840
841 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
842 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
843 P1: InsertVectorEltPred, args: typeIs(TypeIdx: 2, TypesInit: sXLen)))
844 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: InsertVectorEltPred,
845 args: typeIs(TypeIdx: 2, TypesInit: sXLen)));
846
847 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
848 .alwaysLegal();
849
850 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
851
852 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
853
854 verify(MII: *ST.getInstrInfo());
855}
856
857bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
858 MachineInstr &MI) const {
859 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
860
861 if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
862 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {
863 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {
864 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
865 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
866
867 auto OldScalar = MI.getOperand(i: II->ScalarOperand + 2).getReg();
868 // Legalize integer vx form intrinsic.
869 if (MRI.getType(Reg: OldScalar).isScalar()) {
870 if (MRI.getType(Reg: OldScalar).getSizeInBits() < sXLen.getSizeInBits()) {
871 Helper.Observer.changingInstr(MI);
872 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: II->ScalarOperand + 2,
873 ExtOpcode: TargetOpcode::G_ANYEXT);
874 Helper.Observer.changedInstr(MI);
875 } else if (MRI.getType(Reg: OldScalar).getSizeInBits() >
876 sXLen.getSizeInBits()) {
877 // TODO: i64 in riscv32.
878 return false;
879 }
880 }
881 }
882 return true;
883 }
884
885 switch (IntrinsicID) {
886 default:
887 return false;
888 case Intrinsic::riscv_orc_b: {
889 if (!STI.hasStdExtZbb())
890 return false;
891
892 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
893 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
894 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
895 if (Ty == sXLen)
896 return true;
897 if (!STI.is64Bit() || Ty != LLT::scalar(SizeInBits: 32))
898 return false;
899
900 // Each output byte depends only on the corresponding input byte.
901 auto Src = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 2));
902 auto Result = MIRBuilder.buildIntrinsic(ID: IntrinsicID, Res: {DstOp(sXLen)})
903 .addUse(RegNo: Src.getReg(Idx: 0));
904 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: Result);
905 MI.eraseFromParent();
906 return true;
907 }
908 case Intrinsic::riscv_clmulh:
909 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULH, DstOps: {MI.getOperand(i: 0)},
910 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
911 MI.eraseFromParent();
912 return true;
913 case Intrinsic::riscv_clmulr:
914 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULR, DstOps: {MI.getOperand(i: 0)},
915 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
916 MI.eraseFromParent();
917 return true;
918 case Intrinsic::vacopy: {
919 // vacopy arguments must be legal because of the intrinsic signature.
920 // No need to check here.
921
922 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
923 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
924 MachineFunction &MF = *MI.getMF();
925 const DataLayout &DL = MIRBuilder.getDataLayout();
926 LLVMContext &Ctx = MF.getFunction().getContext();
927
928 Register DstLst = MI.getOperand(i: 1).getReg();
929 LLT PtrTy = MRI.getType(Reg: DstLst);
930
931 // Load the source va_list
932 Align Alignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
933 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
934 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: Alignment);
935 auto Tmp = MIRBuilder.buildLoad(Res: PtrTy, Addr: MI.getOperand(i: 2), MMO&: *LoadMMO);
936
937 // Store the result in the destination va_list
938 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
939 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: Alignment);
940 MIRBuilder.buildStore(Val: Tmp, Addr: DstLst, MMO&: *StoreMMO);
941
942 MI.eraseFromParent();
943 return true;
944 }
945 case Intrinsic::riscv_vsetvli:
946 case Intrinsic::riscv_vsetvlimax:
947 case Intrinsic::riscv_masked_atomicrmw_add:
948 case Intrinsic::riscv_masked_atomicrmw_sub:
949 case Intrinsic::riscv_masked_atomicrmw_xchg:
950 case Intrinsic::riscv_masked_atomicrmw_max:
951 case Intrinsic::riscv_masked_atomicrmw_min:
952 case Intrinsic::riscv_masked_atomicrmw_umax:
953 case Intrinsic::riscv_masked_atomicrmw_umin:
954 case Intrinsic::riscv_masked_cmpxchg:
955 return true;
956 }
957}
958
959bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
960 MachineIRBuilder &MIRBuilder) const {
961 // Stores the address of the VarArgsFrameIndex slot into the memory location
962 assert(MI.getOpcode() == TargetOpcode::G_VASTART);
963 MachineFunction *MF = MI.getParent()->getParent();
964 RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>();
965 int FI = FuncInfo->getVarArgsFrameIndex();
966 LLT AddrTy = MIRBuilder.getMRI()->getType(Reg: MI.getOperand(i: 0).getReg());
967 auto FINAddr = MIRBuilder.buildFrameIndex(Res: AddrTy, Idx: FI);
968 assert(MI.hasOneMemOperand());
969 MIRBuilder.buildStore(Val: FINAddr, Addr: MI.getOperand(i: 0).getReg(),
970 MMO&: *MI.memoperands()[0]);
971 MI.eraseFromParent();
972 return true;
973}
974
975bool RISCVLegalizerInfo::legalizeReadCounter(
976 MachineInstr &MI, MachineIRBuilder &MIRBuilder,
977 GISelChangeObserver &Observer) const {
978 assert((MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER ||
979 MI.getOpcode() == TargetOpcode::G_READSTEADYCOUNTER) &&
980 "Unexpected opcode");
981 assert(!STI.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
982 "has custom type legalization on riscv32");
983
984 // On RV32 a 64-bit counter CSR must be read as two 32-bit halves. Because
985 // the count may wrap between the two reads, re-read the high half and loop
986 // until the two high reads agree.
987 int64_t LoCounter, HiCounter;
988 if (MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER) {
989 LoCounter = RISCVSysReg::cycle;
990 HiCounter = RISCVSysReg::cycleh;
991 } else {
992 LoCounter = RISCVSysReg::time;
993 HiCounter = RISCVSysReg::timeh;
994 }
995
996 MachineBasicBlock *BB = MI.getParent();
997 MachineFunction &MF = *BB->getParent();
998 const BasicBlock *LLVMBB = BB->getBasicBlock();
999 DebugLoc DL = MI.getDebugLoc();
1000 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1001
1002 // Split BB into an entry that falls through into a loop block, and a done
1003 // block that receives the remainder of BB and its original successors.
1004 MachineFunction::iterator It = std::next(x: BB->getIterator());
1005 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
1006 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
1007 MF.insert(MBBI: It, MBB: LoopMBB);
1008 MF.insert(MBBI: It, MBB: DoneMBB);
1009
1010 // Splice the instructions after the readcyclecounter into DoneMBB, notifying
1011 // the observer about each moved instruction so CSEInfo stays consistent.
1012 for (MachineBasicBlock::iterator I = std::next(x: MI.getIterator()),
1013 E = BB->end();
1014 I != E; ++I)
1015 Observer.changingInstr(MI&: *I);
1016 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
1017 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
1018 for (MachineInstr &MovedMI : DoneMBB->instrs())
1019 Observer.changedInstr(MI&: MovedMI);
1020 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
1021 BB->addSuccessor(Succ: LoopMBB);
1022
1023 LLT S32 = LLT::scalar(SizeInBits: 32);
1024 // Generic vregs carry the s32 type for G_MERGE_VALUES below, but are also
1025 // constrained to GPR so the target CSRRS/BNE instructions satisfy the
1026 // verifier's register-class constraints.
1027 auto CreateGPR = [&]() {
1028 Register R = MRI.createGenericVirtualRegister(Ty: S32);
1029 MRI.setRegClass(Reg: R, RC: &RISCV::GPRRegClass);
1030 return R;
1031 };
1032 Register LoReg = CreateGPR();
1033 Register HiReg = CreateGPR();
1034 Register ReadAgainReg = CreateGPR();
1035
1036 // read:
1037 // csrrs HiReg, counterh # high word
1038 // csrrs LoReg, counter # low word
1039 // csrrs ReadAgainReg, counterh
1040 // bne HiReg, ReadAgainReg, read
1041 // Emit the target instructions directly with BuildMI.
1042 const RISCVInstrInfo *TII = STI.getInstrInfo();
1043 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
1044 .addImm(Val: HiCounter)
1045 .addReg(RegNo: RISCV::X0);
1046 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
1047 .addImm(Val: LoCounter)
1048 .addReg(RegNo: RISCV::X0);
1049 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
1050 .addImm(Val: HiCounter)
1051 .addReg(RegNo: RISCV::X0);
1052
1053 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
1054 .addReg(RegNo: HiReg)
1055 .addReg(RegNo: ReadAgainReg)
1056 .addMBB(MBB: LoopMBB);
1057
1058 LoopMBB->addSuccessor(Succ: LoopMBB);
1059 LoopMBB->addSuccessor(Succ: DoneMBB);
1060
1061 // Re-pair the two halves into the 64-bit result.
1062 Register DstReg = MI.getOperand(i: 0).getReg();
1063 Observer.erasingInstr(MI);
1064 MI.eraseFromParent();
1065
1066 MIRBuilder.setInsertPt(MBB&: *DoneMBB, II: DoneMBB->begin());
1067 MIRBuilder.setDebugLoc(DL);
1068 MIRBuilder.buildMergeValues(Res: DstReg, Ops: {LoReg, HiReg});
1069 return true;
1070}
1071
1072bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,
1073 MachineIRBuilder &MIRBuilder) const {
1074 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1075 auto &MF = *MI.getParent()->getParent();
1076 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
1077 unsigned EntrySize = MJTI->getEntrySize(TD: MF.getDataLayout());
1078
1079 Register PtrReg = MI.getOperand(i: 0).getReg();
1080 LLT PtrTy = MRI.getType(Reg: PtrReg);
1081 Register IndexReg = MI.getOperand(i: 2).getReg();
1082 LLT IndexTy = MRI.getType(Reg: IndexReg);
1083
1084 if (!isPowerOf2_32(Value: EntrySize))
1085 return false;
1086
1087 auto ShiftAmt = MIRBuilder.buildConstant(Res: IndexTy, Val: Log2_32(Value: EntrySize));
1088 IndexReg = MIRBuilder.buildShl(Dst: IndexTy, Src0: IndexReg, Src1: ShiftAmt).getReg(Idx: 0);
1089
1090 auto Addr = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: IndexReg);
1091
1092 MachineMemOperand *MMO = MF.getMachineMemOperand(
1093 PtrInfo: MachinePointerInfo::getJumpTable(MF), F: MachineMemOperand::MOLoad,
1094 Size: EntrySize, BaseAlignment: Align(MJTI->getEntryAlignment(TD: MF.getDataLayout())));
1095
1096 Register TargetReg;
1097 switch (MJTI->getEntryKind()) {
1098 default:
1099 return false;
1100 case MachineJumpTableInfo::EK_LabelDifference32: {
1101 // For PIC, the sequence is:
1102 // BRIND(load(Jumptable + index) + RelocBase)
1103 // RelocBase can be JumpTable, GOT or some sort of global base.
1104 unsigned LoadOpc =
1105 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;
1106 auto Load = MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: IndexTy, Addr, MMO&: *MMO);
1107 TargetReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: Load).getReg(Idx: 0);
1108 break;
1109 }
1110 case MachineJumpTableInfo::EK_Custom32: {
1111 auto Load = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: IndexTy,
1112 Addr, MMO&: *MMO);
1113 TargetReg = MIRBuilder.buildIntToPtr(Dst: PtrTy, Src: Load).getReg(Idx: 0);
1114 break;
1115 }
1116 case MachineJumpTableInfo::EK_BlockAddress:
1117 TargetReg = MIRBuilder.buildLoad(Res: PtrTy, Addr, MMO&: *MMO).getReg(Idx: 0);
1118 break;
1119 }
1120
1121 if (MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch()) {
1122 // When cf-protection-branch is enabled, we need to use a software
1123 // guarded branch for jump table branch.
1124 MIRBuilder.buildInstr(Opc: RISCV::G_SW_GUARDED_BRIND, DstOps: {}, SrcOps: {TargetReg});
1125 } else {
1126 MIRBuilder.buildBrIndirect(Tgt: TargetReg);
1127 }
1128
1129 MI.eraseFromParent();
1130 return true;
1131}
1132
1133bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,
1134 bool ShouldOptForSize) const {
1135 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);
1136 int64_t Imm = APImm.getSExtValue();
1137 // All simm32 constants should be handled by isel.
1138 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
1139 // this check redundant, but small immediates are common so this check
1140 // should have better compile time.
1141 if (isInt<32>(x: Imm))
1142 return false;
1143
1144 // We only need to cost the immediate, if constant pool lowering is enabled.
1145 if (!STI.useConstantPoolForLargeInts())
1146 return false;
1147
1148 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI);
1149 if (Seq.size() <= STI.getMaxBuildIntsCost())
1150 return false;
1151
1152 // Optimizations below are disabled for opt size. If we're optimizing for
1153 // size, use a constant pool.
1154 if (ShouldOptForSize)
1155 return true;
1156 //
1157 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
1158 // that if it will avoid a constant pool.
1159 // It will require an extra temporary register though.
1160 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
1161 // low and high 32 bits are the same and bit 31 and 63 are set.
1162 unsigned ShiftAmt, AddOpc;
1163 RISCVMatInt::InstSeq SeqLo =
1164 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI, ShiftAmt, AddOpc);
1165 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
1166}
1167
1168bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
1169 MachineIRBuilder &MIB) const {
1170 Register Dst = MI.getOperand(i: 0).getReg();
1171
1172 // We define our scalable vector types for lmul=1 to use a 64 bit known
1173 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
1174 // vscale as VLENB / 8.
1175 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
1176 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1177 // Support for VLEN==32 is incomplete.
1178 return false;
1179
1180 // We assume VLENB is a multiple of 8. We manually choose the best shift
1181 // here because SimplifyDemandedBits isn't always able to simplify it.
1182 uint64_t Val = MI.getOperand(i: 1).getCImm()->getZExtValue();
1183 if (isPowerOf2_64(Value: Val)) {
1184 uint64_t Log2 = Log2_64(Value: Val);
1185 if (Log2 < 3) {
1186 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1187 MIB.buildLShr(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3 - Log2),
1188 Flags: MachineInstr::IsExact);
1189 } else if (Log2 > 3) {
1190 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1191 MIB.buildShl(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Log2 - 3));
1192 } else {
1193 MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {Dst}, SrcOps: {});
1194 }
1195 } else if ((Val % 8) == 0) {
1196 // If the multiplier is a multiple of 8, scale it down to avoid needing
1197 // to shift the VLENB value.
1198 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1199 MIB.buildMul(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Val / 8));
1200 } else {
1201 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1202 auto VScale = MIB.buildLShr(Dst: sXLen, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3),
1203 Flags: MachineInstr::IsExact);
1204 MIB.buildMul(Dst, Src0: VScale, Src1: MIB.buildConstant(Res: sXLen, Val));
1205 }
1206 MI.eraseFromParent();
1207 return true;
1208}
1209
1210// Custom-lower extensions from mask vectors by using a vselect either with 1
1211// for zero/any-extension or -1 for sign-extension:
1212// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
1213// Note that any-extension is lowered identically to zero-extension.
1214bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,
1215 MachineIRBuilder &MIB) const {
1216
1217 unsigned Opc = MI.getOpcode();
1218 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||
1219 Opc == TargetOpcode::G_ANYEXT);
1220
1221 MachineRegisterInfo &MRI = *MIB.getMRI();
1222 Register Dst = MI.getOperand(i: 0).getReg();
1223 Register Src = MI.getOperand(i: 1).getReg();
1224
1225 LLT DstTy = MRI.getType(Reg: Dst);
1226 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;
1227 LLT DstEltTy = DstTy.getElementType();
1228 auto SplatZero = MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: 0));
1229 auto SplatTrue =
1230 MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: ExtTrueVal));
1231 MIB.buildSelect(Res: Dst, Tst: Src, Op0: SplatTrue, Op1: SplatZero);
1232
1233 MI.eraseFromParent();
1234 return true;
1235}
1236
1237bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,
1238 LegalizerHelper &Helper,
1239 MachineIRBuilder &MIB) const {
1240 assert((isa<GLoad>(MI) || isa<GStore>(MI)) &&
1241 "Machine instructions must be Load/Store.");
1242 MachineRegisterInfo &MRI = *MIB.getMRI();
1243 MachineFunction *MF = MI.getMF();
1244 const DataLayout &DL = MIB.getDataLayout();
1245 LLVMContext &Ctx = MF->getFunction().getContext();
1246
1247 Register DstReg = MI.getOperand(i: 0).getReg();
1248 LLT DataTy = MRI.getType(Reg: DstReg);
1249 if (!DataTy.isVector())
1250 return false;
1251
1252 if (!MI.hasOneMemOperand())
1253 return false;
1254
1255 MachineMemOperand *MMO = *MI.memoperands_begin();
1256
1257 const auto *TLI = STI.getTargetLowering();
1258 EVT VT = EVT::getEVT(Ty: getTypeForLLT(Ty: DataTy, C&: Ctx));
1259
1260 if (TLI->allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT, MMO: *MMO))
1261 return true;
1262
1263 unsigned EltSizeBits = DataTy.getScalarSizeInBits();
1264 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
1265 "Unexpected unaligned RVV load type");
1266
1267 // Calculate the new vector type with i8 elements
1268 unsigned NumElements =
1269 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);
1270 LLT NewDataTy = LLT::scalable_vector(MinNumElements: NumElements, ScalarSizeInBits: 8);
1271
1272 Helper.bitcast(MI, TypeIdx: 0, Ty: NewDataTy);
1273
1274 return true;
1275}
1276
1277/// Return the type of the mask type suitable for masking the provided
1278/// vector type. This is simply an i1 element type vector of the same
1279/// (possibly scalable) length.
1280static LLT getMaskTypeFor(LLT VecTy) {
1281 assert(VecTy.isVector());
1282 ElementCount EC = VecTy.getElementCount();
1283 return LLT::vector(EC, ScalarTy: LLT::scalar(SizeInBits: 1));
1284}
1285
1286/// Creates an all ones mask suitable for masking a vector of type VecTy with
1287/// vector length VL.
1288static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,
1289 MachineIRBuilder &MIB,
1290 MachineRegisterInfo &MRI) {
1291 LLT MaskTy = getMaskTypeFor(VecTy);
1292 return MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {MaskTy}, SrcOps: {VL});
1293}
1294
1295/// Gets the two common "VL" operands: an all-ones mask and the vector length.
1296/// VecTy is a scalable vector type.
1297static std::pair<MachineInstrBuilder, MachineInstrBuilder>
1298buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1299 assert(VecTy.isScalableVector() && "Expecting scalable container type");
1300 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
1301 LLT XLenTy(STI.getXLenVT());
1302 auto VL = MIB.buildConstant(Res: XLenTy, Val: -1);
1303 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
1304 return {Mask, VL};
1305}
1306
1307static MachineInstrBuilder
1308buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
1309 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
1310 MachineRegisterInfo &MRI) {
1311 // TODO: If the Hi bits of the splat are undefined, then it's fine to just
1312 // splat Lo even if it might be sign extended. I don't think we have
1313 // introduced a case where we're build a s64 where the upper bits are undef
1314 // yet.
1315
1316 // Fall back to a stack store and stride x0 vector load.
1317 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in
1318 // preprocessDAG in SDAG.
1319 return MIB.buildInstr(Opc: RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, DstOps: {Dst},
1320 SrcOps: {Passthru, Lo, Hi, VL});
1321}
1322
1323static MachineInstrBuilder
1324buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
1325 const SrcOp &Scalar, const SrcOp &VL,
1326 MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1327 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
1328 auto Unmerge = MIB.buildUnmerge(Res: LLT::scalar(SizeInBits: 32), Op: Scalar);
1329 return buildSplatPartsS64WithVL(Dst, Passthru, Lo: Unmerge.getReg(Idx: 0),
1330 Hi: Unmerge.getReg(Idx: 1), VL, MIB, MRI);
1331}
1332
1333// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a
1334// legal equivalently-sized i8 type, so we can use that as a go-between.
1335// Splats of s1 types that have constant value can be legalized as VMSET_VL or
1336// VMCLR_VL.
1337bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
1338 MachineIRBuilder &MIB) const {
1339 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);
1340
1341 MachineRegisterInfo &MRI = *MIB.getMRI();
1342
1343 Register Dst = MI.getOperand(i: 0).getReg();
1344 Register SplatVal = MI.getOperand(i: 1).getReg();
1345
1346 LLT VecTy = MRI.getType(Reg: Dst);
1347 LLT XLenTy(STI.getXLenVT());
1348
1349 // Handle case of s64 element vectors on rv32
1350 if (XLenTy.getSizeInBits() == 32 &&
1351 VecTy.getElementType().getSizeInBits() == 64) {
1352 auto [_, VL] = buildDefaultVLOps(VecTy: MRI.getType(Reg: Dst), MIB, MRI);
1353 buildSplatSplitS64WithVL(Dst, Passthru: MIB.buildUndef(Res: VecTy), Scalar: SplatVal, VL, MIB,
1354 MRI);
1355 MI.eraseFromParent();
1356 return true;
1357 }
1358
1359 // All-zeros or all-ones splats are handled specially.
1360 MachineInstr &SplatValMI = *MRI.getVRegDef(Reg: SplatVal);
1361 if (isAllOnesOrAllOnesSplat(MI: SplatValMI, MRI)) {
1362 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1363 MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {Dst}, SrcOps: {VL});
1364 MI.eraseFromParent();
1365 return true;
1366 }
1367 if (isNullOrNullSplat(MI: SplatValMI, MRI)) {
1368 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1369 MIB.buildInstr(Opc: RISCV::G_VMCLR_VL, DstOps: {Dst}, SrcOps: {VL});
1370 MI.eraseFromParent();
1371 return true;
1372 }
1373
1374 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all
1375 // ones) by promoting it to an s8 splat.
1376 LLT InterEltTy = LLT::scalar(SizeInBits: 8);
1377 LLT InterTy = VecTy.changeElementType(NewEltTy: InterEltTy);
1378 auto ZExtSplatVal = MIB.buildZExt(Res: InterEltTy, Op: SplatVal);
1379 auto And =
1380 MIB.buildAnd(Dst: InterEltTy, Src0: ZExtSplatVal, Src1: MIB.buildConstant(Res: InterEltTy, Val: 1));
1381 auto LHS = MIB.buildSplatVector(Res: InterTy, Val: And);
1382 auto ZeroSplat =
1383 MIB.buildSplatVector(Res: InterTy, Val: MIB.buildConstant(Res: InterEltTy, Val: 0));
1384 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: LHS, Op1: ZeroSplat);
1385 MI.eraseFromParent();
1386 return true;
1387}
1388
1389static LLT getLMUL1Ty(LLT VecTy) {
1390 assert(VecTy.getElementType().getSizeInBits() <= 64 &&
1391 "Unexpected vector LLT");
1392 return LLT::scalable_vector(MinNumElements: RISCV::RVVBitsPerBlock /
1393 VecTy.getElementType().getSizeInBits(),
1394 ScalarTy: VecTy.getElementType());
1395}
1396
1397bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
1398 MachineIRBuilder &MIB) const {
1399 GExtractSubvector &ES = cast<GExtractSubvector>(Val&: MI);
1400
1401 MachineRegisterInfo &MRI = *MIB.getMRI();
1402
1403 Register Dst = ES.getReg(Idx: 0);
1404 Register Src = ES.getSrcVec();
1405 uint64_t Idx = ES.getIndexImm();
1406
1407 // With an index of 0 this is a cast-like subvector, which can be performed
1408 // with subregister operations.
1409 if (Idx == 0)
1410 return true;
1411
1412 LLT LitTy = MRI.getType(Reg: Dst);
1413 LLT BigTy = MRI.getType(Reg: Src);
1414
1415 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1416 // We can't slide this mask vector up indexed by its i1 elements.
1417 // This poses a problem when we wish to insert a scalable vector which
1418 // can't be re-expressed as a larger type. Just choose the slow path and
1419 // extend to a larger type, then truncate back down.
1420 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1421 LLT ExtLitTy = LitTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1422 auto BigZExt = MIB.buildZExt(Res: ExtBigTy, Op: Src);
1423 auto ExtractZExt = MIB.buildExtractSubvector(Res: ExtLitTy, Src: BigZExt, Index: Idx);
1424 auto SplatZero = MIB.buildSplatVector(
1425 Res: ExtLitTy, Val: MIB.buildConstant(Res: ExtLitTy.getElementType(), Val: 0));
1426 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: ExtractZExt, Op1: SplatZero);
1427 MI.eraseFromParent();
1428 return true;
1429 }
1430
1431 // extract_subvector scales the index by vscale if the subvector is scalable,
1432 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
1433 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1434 MVT LitTyMVT = getMVTForLLT(Ty: LitTy);
1435 auto Decompose =
1436 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1437 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: LitTyMVT, InsertExtractIdx: Idx, TRI);
1438 unsigned RemIdx = Decompose.second;
1439
1440 // If the Idx has been completely eliminated then this is a subvector extract
1441 // which naturally aligns to a vector register. These can easily be handled
1442 // using subregister manipulation.
1443 if (RemIdx == 0)
1444 return true;
1445
1446 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
1447 // was > M1 then the index would need to be a multiple of VLMAX, and so would
1448 // divide exactly.
1449 assert(
1450 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||
1451 RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVVType::LMUL_1);
1452
1453 // If the vector type is an LMUL-group type, extract a subvector equal to the
1454 // nearest full vector register type.
1455 LLT InterLitTy = BigTy;
1456 Register Vec = Src;
1457 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1458 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1459 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
1460 // we should have successfully decomposed the extract into a subregister.
1461 assert(Decompose.first != RISCV::NoSubRegister);
1462 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1463 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
1464 // specified on the source Register (the equivalent) since generic virtual
1465 // register does not allow subregister index.
1466 Vec = MIB.buildExtractSubvector(Res: InterLitTy, Src, Index: Idx - RemIdx).getReg(Idx: 0);
1467 }
1468
1469 // Slide this vector register down by the desired number of elements in order
1470 // to place the desired subvector starting at element 0.
1471 const LLT XLenTy(STI.getXLenVT());
1472 auto SlidedownAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1473 auto [Mask, VL] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1474 uint64_t Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
1475 auto Slidedown = MIB.buildInstr(
1476 Opc: RISCV::G_VSLIDEDOWN_VL, DstOps: {InterLitTy},
1477 SrcOps: {MIB.buildUndef(Res: InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
1478
1479 // Now the vector is in the right position, extract our final subvector. This
1480 // should resolve to a COPY.
1481 MIB.buildExtractSubvector(Res: Dst, Src: Slidedown, Index: 0);
1482
1483 MI.eraseFromParent();
1484 return true;
1485}
1486
1487bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,
1488 LegalizerHelper &Helper,
1489 MachineIRBuilder &MIB) const {
1490 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
1491
1492 MachineRegisterInfo &MRI = *MIB.getMRI();
1493
1494 Register Dst = IS.getReg(Idx: 0);
1495 Register BigVec = IS.getBigVec();
1496 Register LitVec = IS.getSubVec();
1497 uint64_t Idx = IS.getIndexImm();
1498
1499 LLT BigTy = MRI.getType(Reg: BigVec);
1500 LLT LitTy = MRI.getType(Reg: LitVec);
1501
1502 if (Idx == 0 && mi_match(R: BigVec, MRI, P: m_GImplicitDef()))
1503 return true;
1504
1505 // We don't have the ability to slide mask vectors up indexed by their i1
1506 // elements; the smallest we can do is i8. Often we are able to bitcast to
1507 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i8
1508 // vectors and truncate down after the insert.
1509 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1510 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();
1511 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();
1512 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)
1513 return Helper.bitcast(
1514 MI&: IS, TypeIdx: 0,
1515 Ty: LLT::vector(EC: BigTy.getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8));
1516
1517 // We can't slide this mask vector up indexed by its i1 elements.
1518 // This poses a problem when we wish to insert a scalable vector which
1519 // can't be re-expressed as a larger type. Just choose the slow path and
1520 // extend to a larger type, then truncate back down.
1521 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1522 return Helper.widenScalar(MI&: IS, TypeIdx: 0, WideTy: ExtBigTy);
1523 }
1524
1525 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1526 unsigned SubRegIdx, RemIdx;
1527 std::tie(args&: SubRegIdx, args&: RemIdx) =
1528 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1529 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: getMVTForLLT(Ty: LitTy), InsertExtractIdx: Idx, TRI);
1530
1531 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
1532 assert(isPowerOf2_64(
1533 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));
1534 bool ExactlyVecRegSized =
1535 STI.expandVScale(X: LitTy.getSizeInBits())
1536 .isKnownMultipleOf(RHS: STI.expandVScale(X: VecRegSize));
1537
1538 // If the Idx has been completely eliminated and this subvector's size is a
1539 // vector register or a multiple thereof, or the surrounding elements are
1540 // undef, then this is a subvector insert which naturally aligns to a vector
1541 // register. These can easily be handled using subregister manipulation.
1542 if (RemIdx == 0 && ExactlyVecRegSized)
1543 return true;
1544
1545 // If the subvector is smaller than a vector register, then the insertion
1546 // must preserve the undisturbed elements of the register. We do this by
1547 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
1548 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
1549 // subvector within the vector register, and an INSERT_SUBVECTOR of that
1550 // LMUL=1 type back into the larger vector (resolving to another subregister
1551 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
1552 // to avoid allocating a large register group to hold our subvector.
1553
1554 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
1555 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
1556 // (in our case undisturbed). This means we can set up a subvector insertion
1557 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
1558 // size of the subvector.
1559 const LLT XLenTy(STI.getXLenVT());
1560 LLT InterLitTy = BigTy;
1561 Register AlignedExtract = BigVec;
1562 unsigned AlignedIdx = Idx - RemIdx;
1563 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1564 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1565 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1566 // Extract a subvector equal to the nearest full vector register type. This
1567 // should resolve to a G_EXTRACT on a subreg.
1568 AlignedExtract =
1569 MIB.buildExtractSubvector(Res: InterLitTy, Src: BigVec, Index: AlignedIdx).getReg(Idx: 0);
1570 }
1571
1572 auto Insert = MIB.buildInsertSubvector(Res: InterLitTy, Src0: MIB.buildUndef(Res: InterLitTy),
1573 Src1: LitVec, Index: 0);
1574
1575 auto [Mask, _] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1576 auto VL = MIB.buildVScale(Res: XLenTy, MinElts: LitTy.getElementCount().getKnownMinValue());
1577
1578 // If we're inserting into the lowest elements, use a tail undisturbed
1579 // vmv.v.v.
1580 MachineInstrBuilder Inserted;
1581 bool NeedInsertSubvec =
1582 TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(), RHS: InterLitTy.getSizeInBits());
1583 Register InsertedDst =
1584 NeedInsertSubvec ? MRI.createGenericVirtualRegister(Ty: InterLitTy) : Dst;
1585 if (RemIdx == 0) {
1586 Inserted = MIB.buildInstr(Opc: RISCV::G_VMV_V_V_VL, DstOps: {InsertedDst},
1587 SrcOps: {AlignedExtract, Insert, VL});
1588 } else {
1589 auto SlideupAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1590 // Construct the vector length corresponding to RemIdx + length(LitTy).
1591 VL = MIB.buildAdd(Dst: XLenTy, Src0: SlideupAmt, Src1: VL);
1592 // Use tail agnostic policy if we're inserting over InterLitTy's tail.
1593 ElementCount EndIndex =
1594 ElementCount::getScalable(MinVal: RemIdx) + LitTy.getElementCount();
1595 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
1596 if (STI.expandVScale(X: EndIndex) ==
1597 STI.expandVScale(X: InterLitTy.getElementCount()))
1598 Policy = RISCVVType::TAIL_AGNOSTIC;
1599
1600 Inserted =
1601 MIB.buildInstr(Opc: RISCV::G_VSLIDEUP_VL, DstOps: {InsertedDst},
1602 SrcOps: {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});
1603 }
1604
1605 // If required, insert this subvector back into the correct vector register.
1606 // This should resolve to an INSERT_SUBREG instruction.
1607 if (NeedInsertSubvec)
1608 MIB.buildInsertSubvector(Res: Dst, Src0: BigVec, Src1: Inserted, Index: AlignedIdx);
1609
1610 MI.eraseFromParent();
1611 return true;
1612}
1613
1614bool RISCVLegalizerInfo::legalizeBitreverse(MachineInstr &MI,
1615 MachineIRBuilder &MIB) const {
1616 assert(MI.getOpcode() == TargetOpcode::G_BITREVERSE && "Unexpected opcode");
1617
1618 if (!STI.hasStdExtZbkb())
1619 return false;
1620
1621 MachineRegisterInfo &MRI = *MIB.getMRI();
1622
1623 Register Dst = MI.getOperand(i: 0).getReg();
1624 Register Src = MI.getOperand(i: 1).getReg();
1625
1626 if (!MRI.getType(Reg: Dst).isScalar(Size: 8))
1627 return false;
1628
1629 auto WideSrc = MIB.buildAnyExt(Res: sXLen, Op: Src);
1630 auto Brev = MIB.buildInstr(Opc: RISCV::G_BREV8, DstOps: {sXLen}, SrcOps: {WideSrc.getReg(Idx: 0)});
1631 MIB.buildTrunc(Res: Dst, Op: Brev.getReg(Idx: 0));
1632
1633 MI.eraseFromParent();
1634 return true;
1635}
1636
1637static unsigned getRISCVWOpcode(unsigned Opcode) {
1638 switch (Opcode) {
1639 default:
1640 llvm_unreachable("Unexpected opcode");
1641 case TargetOpcode::G_ASHR:
1642 return RISCV::G_SRAW;
1643 case TargetOpcode::G_LSHR:
1644 return RISCV::G_SRLW;
1645 case TargetOpcode::G_SHL:
1646 return RISCV::G_SLLW;
1647 case TargetOpcode::G_SDIV:
1648 return RISCV::G_DIVW;
1649 case TargetOpcode::G_UDIV:
1650 return RISCV::G_DIVUW;
1651 case TargetOpcode::G_UREM:
1652 return RISCV::G_REMUW;
1653 case TargetOpcode::G_ROTL:
1654 return RISCV::G_ROLW;
1655 case TargetOpcode::G_ROTR:
1656 return RISCV::G_RORW;
1657 case TargetOpcode::G_CTLZ:
1658 return RISCV::G_CLZW;
1659 case TargetOpcode::G_CTTZ:
1660 return RISCV::G_CTZW;
1661 case TargetOpcode::G_CTLS:
1662 return RISCV::G_CLSW;
1663 case TargetOpcode::G_FPTOSI:
1664 return RISCV::G_FCVT_W_RV64;
1665 case TargetOpcode::G_FPTOUI:
1666 return RISCV::G_FCVT_WU_RV64;
1667 }
1668}
1669
1670bool RISCVLegalizerInfo::legalizeCustom(
1671 LegalizerHelper &Helper, MachineInstr &MI,
1672 LostDebugLocObserver &LocObserver) const {
1673 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1674 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1675 MachineFunction &MF = *MI.getParent()->getParent();
1676 switch (MI.getOpcode()) {
1677 default:
1678 // No idea what to do.
1679 return false;
1680 case TargetOpcode::G_ABS:
1681 return Helper.lowerAbsToMaxNeg(MI);
1682 case TargetOpcode::G_CLMULH:
1683 case TargetOpcode::G_CLMULR: {
1684 assert(STI.is64Bit() &&
1685 MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32) &&
1686 "Unexpected custom legalization");
1687 // Shift both inputs by 32 so the full product has 64 trailing zeros.
1688 // Perform CLMULH or CLMULR on the shifted inputs, then extract the upper
1689 // 32 bits of the result.
1690 auto Shift = MIRBuilder.buildConstant(Res: sXLen, Val: 32);
1691 auto LHS = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 1));
1692 auto RHS = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 2));
1693 auto ShiftedLHS = MIRBuilder.buildShl(Dst: sXLen, Src0: LHS, Src1: Shift);
1694 auto ShiftedRHS = MIRBuilder.buildShl(Dst: sXLen, Src0: RHS, Src1: Shift);
1695 auto Product = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {sXLen},
1696 SrcOps: {ShiftedLHS, ShiftedRHS});
1697 auto High = MIRBuilder.buildLShr(Dst: sXLen, Src0: Product, Src1: Shift);
1698 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: High);
1699 MI.eraseFromParent();
1700 return true;
1701 }
1702 case TargetOpcode::G_FCONSTANT: {
1703 const APFloat &FVal = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1704
1705 // Convert G_FCONSTANT to G_CONSTANT.
1706 Register DstReg = MI.getOperand(i: 0).getReg();
1707 MIRBuilder.buildConstant(Res: DstReg, Val: FVal.bitcastToAPInt());
1708
1709 MI.eraseFromParent();
1710 return true;
1711 }
1712 case TargetOpcode::G_CONSTANT: {
1713 const Function &F = MF.getFunction();
1714 // TODO: if PSI and BFI are present, add " ||
1715 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".
1716 bool ShouldOptForSize = F.hasOptSize();
1717 const ConstantInt *ConstVal = MI.getOperand(i: 1).getCImm();
1718 if (!shouldBeInConstantPool(APImm: ConstVal->getValue(), ShouldOptForSize))
1719 return true;
1720 return Helper.lowerConstant(MI);
1721 }
1722 case TargetOpcode::G_SUB:
1723 case TargetOpcode::G_ADD: {
1724 Helper.Observer.changingInstr(MI);
1725 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1726 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1727
1728 Register DstALU = MRI.createGenericVirtualRegister(Ty: sXLen);
1729
1730 MachineOperand &MO = MI.getOperand(i: 0);
1731 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
1732 auto DstSext = MIRBuilder.buildSExtInReg(Res: sXLen, Op: DstALU, ImmOp: 32);
1733
1734 MIRBuilder.buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: {MO}, SrcOps: {DstSext});
1735 MO.setReg(DstALU);
1736
1737 Helper.Observer.changedInstr(MI);
1738 return true;
1739 }
1740 case TargetOpcode::G_ASHR:
1741 case TargetOpcode::G_LSHR:
1742 case TargetOpcode::G_SHL: {
1743 if (getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI)) {
1744 // We don't need a custom node for shift by constant. Just widen the
1745 // source and the shift amount.
1746 unsigned ExtOpc = TargetOpcode::G_ANYEXT;
1747 if (MI.getOpcode() == TargetOpcode::G_ASHR)
1748 ExtOpc = TargetOpcode::G_SEXT;
1749 else if (MI.getOpcode() == TargetOpcode::G_LSHR)
1750 ExtOpc = TargetOpcode::G_ZEXT;
1751
1752 Helper.Observer.changingInstr(MI);
1753 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: ExtOpc);
1754 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
1755 Helper.widenScalarDst(MI, WideTy: sXLen);
1756 Helper.Observer.changedInstr(MI);
1757 return true;
1758 }
1759
1760 Helper.Observer.changingInstr(MI);
1761 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1762 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1763 Helper.widenScalarDst(MI, WideTy: sXLen);
1764 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1765 Helper.Observer.changedInstr(MI);
1766 return true;
1767 }
1768 case TargetOpcode::G_SDIV:
1769 case TargetOpcode::G_UDIV:
1770 case TargetOpcode::G_UREM:
1771 case TargetOpcode::G_ROTL:
1772 case TargetOpcode::G_ROTR: {
1773 Helper.Observer.changingInstr(MI);
1774 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1775 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1776 Helper.widenScalarDst(MI, WideTy: sXLen);
1777 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1778 Helper.Observer.changedInstr(MI);
1779 return true;
1780 }
1781 case TargetOpcode::G_CTLZ:
1782 case TargetOpcode::G_CTTZ:
1783 case TargetOpcode::G_CTLS: {
1784 Helper.Observer.changingInstr(MI);
1785 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1786 Helper.widenScalarDst(MI, WideTy: sXLen);
1787 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1788 Helper.Observer.changedInstr(MI);
1789 return true;
1790 }
1791 case TargetOpcode::G_FPTOSI:
1792 case TargetOpcode::G_FPTOUI: {
1793 Helper.Observer.changingInstr(MI);
1794 Helper.widenScalarDst(MI, WideTy: sXLen);
1795 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1796 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RTZ));
1797 Helper.Observer.changedInstr(MI);
1798 return true;
1799 }
1800 case TargetOpcode::G_LROUND: {
1801 // The (i32 any_lround) Pat is IsRV32-only; on RV64 lower to
1802 // riscv_fcvt_w_rv64 with FRM_RMM.
1803 Helper.Observer.changingInstr(MI);
1804 Helper.widenScalarDst(MI, WideTy: sXLen);
1805 MI.setDesc(MIRBuilder.getTII().get(Opcode: RISCV::G_FCVT_W_RV64));
1806 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RMM));
1807 Helper.Observer.changedInstr(MI);
1808 return true;
1809 }
1810 case TargetOpcode::G_READCYCLECOUNTER:
1811 case TargetOpcode::G_READSTEADYCOUNTER:
1812 return legalizeReadCounter(MI, MIRBuilder, Observer&: Helper.Observer);
1813 case TargetOpcode::G_IS_FPCLASS: {
1814 Register GISFPCLASS = MI.getOperand(i: 0).getReg();
1815 Register Src = MI.getOperand(i: 1).getReg();
1816 const MachineOperand &ImmOp = MI.getOperand(i: 2);
1817 MachineIRBuilder MIB(MI);
1818
1819 // Turn LLVM IR's floating point classes to that in RISC-V,
1820 // by simply rotating the 10-bit immediate right by two bits.
1821 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));
1822 auto FClassMask = MIB.buildConstant(Res: sXLen, Val: GFpClassImm.rotr(rotateAmt: 2).zext(width: XLen));
1823 auto ConstZero = MIB.buildConstant(Res: sXLen, Val: 0);
1824
1825 auto GFClass = MIB.buildInstr(Opc: RISCV::G_FCLASS, DstOps: {sXLen}, SrcOps: {Src});
1826 auto And = MIB.buildAnd(Dst: sXLen, Src0: GFClass, Src1: FClassMask);
1827 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: GISFPCLASS, Op0: And, Op1: ConstZero);
1828
1829 MI.eraseFromParent();
1830 return true;
1831 }
1832 case TargetOpcode::G_BRJT:
1833 return legalizeBRJT(MI, MIRBuilder);
1834 case TargetOpcode::G_VASTART:
1835 return legalizeVAStart(MI, MIRBuilder);
1836 case TargetOpcode::G_VSCALE:
1837 return legalizeVScale(MI, MIB&: MIRBuilder);
1838 case TargetOpcode::G_ZEXT:
1839 case TargetOpcode::G_SEXT:
1840 case TargetOpcode::G_ANYEXT:
1841 return legalizeExt(MI, MIB&: MIRBuilder);
1842 case TargetOpcode::G_SPLAT_VECTOR:
1843 return legalizeSplatVector(MI, MIB&: MIRBuilder);
1844 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1845 return legalizeExtractSubvector(MI, MIB&: MIRBuilder);
1846 case TargetOpcode::G_INSERT_SUBVECTOR:
1847 return legalizeInsertSubvector(MI, Helper, MIB&: MIRBuilder);
1848 case TargetOpcode::G_BITREVERSE:
1849 return legalizeBitreverse(MI, MIB&: MIRBuilder);
1850 case TargetOpcode::G_LOAD:
1851 case TargetOpcode::G_STORE:
1852 return legalizeLoadStore(MI, Helper, MIB&: MIRBuilder);
1853 }
1854
1855 llvm_unreachable("expected switch to return");
1856}
1857