1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for RISC-V.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "RISCVLegalizerInfo.h"
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVSubtarget.h"
17#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/MachineConstantPool.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/TargetOpcodes.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsRISCV.h"
32#include "llvm/IR/Type.h"
33
34using namespace llvm;
35using namespace LegalityPredicates;
36using namespace LegalizeMutations;
37using namespace MIPatternMatch;
38
39static LegalityPredicate
40typeIsLegalIntOrFPVec(unsigned TypeIdx,
41 std::initializer_list<LLT> IntOrFPVecTys,
42 const RISCVSubtarget &ST) {
43 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
44 return ST.hasVInstructions() &&
45 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
46 ST.hasVInstructionsI64()) &&
47 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
48 ST.getELen() == 64);
49 };
50
51 return all(P0: typeInSet(TypeIdx, TypesInit: IntOrFPVecTys), P1: P);
52}
53
54static LegalityPredicate
55typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
56 const RISCVSubtarget &ST) {
57 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
58 return ST.hasVInstructions() &&
59 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
60 ST.getELen() == 64);
61 };
62 return all(P0: typeInSet(TypeIdx, TypesInit: BoolVecTys), P1: P);
63}
64
65static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
66 std::initializer_list<LLT> PtrVecTys,
67 const RISCVSubtarget &ST) {
68 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
69 return ST.hasVInstructions() &&
70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
71 ST.getELen() == 64) &&
72 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
73 Query.Types[TypeIdx].getScalarSizeInBits() == 32);
74 };
75 return all(P0: typeInSet(TypeIdx, TypesInit: PtrVecTys), P1: P);
76}
77
78RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
79 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(SizeInBits: XLen)) {
80 const LLT sDoubleXLen = LLT::scalar(SizeInBits: 2 * XLen);
81 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: XLen);
82 const LLT s1 = LLT::scalar(SizeInBits: 1);
83 const LLT s8 = LLT::scalar(SizeInBits: 8);
84 const LLT s16 = LLT::scalar(SizeInBits: 16);
85 const LLT f16 = LLT::float16();
86 const LLT s32 = LLT::scalar(SizeInBits: 32);
87 const LLT s64 = LLT::scalar(SizeInBits: 64);
88 const LLT s128 = LLT::scalar(SizeInBits: 128);
89
90 const LLT nxv1s1 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s1);
91 const LLT nxv2s1 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s1);
92 const LLT nxv4s1 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s1);
93 const LLT nxv8s1 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s1);
94 const LLT nxv16s1 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s1);
95 const LLT nxv32s1 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s1);
96 const LLT nxv64s1 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s1);
97
98 const LLT nxv1s8 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s8);
99 const LLT nxv2s8 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s8);
100 const LLT nxv4s8 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s8);
101 const LLT nxv8s8 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s8);
102 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
103 const LLT nxv32s8 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s8);
104 const LLT nxv64s8 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s8);
105
106 const LLT nxv1s16 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s16);
107 const LLT nxv2s16 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s16);
108 const LLT nxv4s16 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s16);
109 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
110 const LLT nxv16s16 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s16);
111 const LLT nxv32s16 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s16);
112
113 const LLT nxv1s32 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s32);
114 const LLT nxv2s32 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s32);
115 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
116 const LLT nxv8s32 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s32);
117 const LLT nxv16s32 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s32);
118
119 const LLT nxv1s64 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s64);
120 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
121 const LLT nxv4s64 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s64);
122 const LLT nxv8s64 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s64);
123
124 const LLT nxv1p0 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: p0);
125 const LLT nxv2p0 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: p0);
126 const LLT nxv4p0 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: p0);
127 const LLT nxv8p0 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: p0);
128 const LLT nxv16p0 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: p0);
129
130 using namespace TargetOpcode;
131
132 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
133
134 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,
135 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
136 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
137 nxv1s64, nxv2s64, nxv4s64, nxv8s64};
138
139 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
140
141 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
142 .legalFor(Types: {sXLen})
143 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
144 .customFor(Pred: ST.is64Bit(), Types: {s32})
145 .widenScalarToNextPow2(TypeIdx: 0)
146 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
147
148 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
149 .legalFor(Types: {sXLen})
150 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
151 .widenScalarToNextPow2(TypeIdx: 0)
152 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
153
154 getActionDefinitionsBuilder(
155 Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO, G_READ_REGISTER, G_WRITE_REGISTER})
156 .lower();
157
158 getActionDefinitionsBuilder(Opcodes: {G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})
159 .minScalar(TypeIdx: 0, Ty: sXLen)
160 .lower();
161
162 // TODO: Use Vector Single-Width Saturating Instructions for vector types.
163 getActionDefinitionsBuilder(
164 Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})
165 .lower();
166
167 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
168 .legalFor(Types: {{sXLen, sXLen}})
169 .customFor(Pred: ST.is64Bit(), Types: {{s32, s32}})
170 .widenScalarToNextPow2(TypeIdx: 0)
171 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
172 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
173
174 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
175 .legalFor(Types: {{s32, s16}})
176 .legalFor(Pred: ST.is64Bit(), Types: {{s64, s16}, {s64, s32}})
177 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
178 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
179 .customIf(Predicate: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))
180 .maxScalar(TypeIdx: 0, Ty: sXLen);
181
182 getActionDefinitionsBuilder(Opcode: G_TRUNC).alwaysLegal();
183
184 {
185 LegalityPredicate ValidSextInRegWidth = all(P0: sizeIs(TypeIdx: 0, Size: 64), P1: immIs(ImmIdx: 0, Imm: 32));
186
187 if (STI.hasStdExtZbb())
188 ValidSextInRegWidth =
189 LegalityPredicates::any(P0: ValidSextInRegWidth, P1: immInSet(ImmIdx: 0, ImmsInit: {8, 16}));
190
191 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
192 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: sXLen), P1: ValidSextInRegWidth))
193 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
194 .lower();
195 }
196
197 // Merge/Unmerge
198 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
199 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Opcode: Op);
200 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
201 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
202 if (XLen == 32 && ST.hasStdExtD()) {
203 MergeUnmergeActions.legalIf(
204 Predicate: all(P0: typeIs(TypeIdx: BigTyIdx, TypesInit: s64), P1: typeIs(TypeIdx: LitTyIdx, TypesInit: s32)));
205 }
206 MergeUnmergeActions.widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: XLen)
207 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: XLen)
208 .clampScalar(TypeIdx: LitTyIdx, MinTy: sXLen, MaxTy: sXLen)
209 .clampScalar(TypeIdx: BigTyIdx, MinTy: sXLen, MaxTy: sXLen);
210 }
211
212 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR}).lower();
213
214 getActionDefinitionsBuilder(Opcodes: {G_ROTR, G_ROTL})
215 .legalFor(Pred: ST.hasStdExtZbb() || ST.hasStdExtZbkb(), Types: {{sXLen, sXLen}})
216 .customFor(Pred: ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),
217 Types: {{s32, s32}})
218 .lower();
219
220 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
221 .customFor(Pred: ST.hasStdExtZbkb(), Types: {s8})
222 .maxScalar(TypeIdx: 0, Ty: sXLen)
223 .lower();
224
225 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
226 Predicate: all(P0: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
227 P1: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST)),
228 P1: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST),
229 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))));
230
231 auto &BSWAPActions = getActionDefinitionsBuilder(Opcode: G_BSWAP);
232 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
233 BSWAPActions.legalFor(Types: {sXLen}).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
234 else
235 BSWAPActions.maxScalar(TypeIdx: 0, Ty: sXLen).lower();
236
237 getActionDefinitionsBuilder(Opcode: G_CLMUL)
238 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
239 .unsupported();
240
241 getActionDefinitionsBuilder(Opcode: G_CLMULH)
242 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
243 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbkc(), Types: {s32})
244 .unsupported();
245
246 // CLMULR is Zbc-only; Zbkc is a subset that has CLMUL/CLMULH but not CLMULR.
247 getActionDefinitionsBuilder(Opcode: G_CLMULR)
248 .legalFor(Pred: ST.hasStdExtZbc(), Types: {sXLen})
249 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbc(), Types: {s32})
250 .unsupported();
251
252 auto &CountZerosActions = getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTTZ});
253 auto &CountZerosPoisonActions =
254 getActionDefinitionsBuilder(Opcodes: {G_CTLZ_ZERO_POISON, G_CTTZ_ZERO_POISON});
255 if (ST.hasStdExtZbb()) {
256 CountZerosActions.legalFor(Types: {{sXLen, sXLen}})
257 .customFor(Types: {{s32, s32}})
258 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
259 .widenScalarToNextPow2(TypeIdx: 0)
260 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
261 } else {
262 CountZerosActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
263 CountZerosPoisonActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
264 }
265 CountZerosPoisonActions.lower();
266
267 auto &CountSignActions = getActionDefinitionsBuilder(Opcode: G_CTLS);
268 if (ST.hasStdExtP()) {
269 CountSignActions.legalFor(Types: {{sXLen, sXLen}})
270 .customFor(Types: {{s32, s32}})
271 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
272 .widenScalarToNextPow2(TypeIdx: 0)
273 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
274 } else {
275 CountSignActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
276 }
277
278 auto &CTPOPActions = getActionDefinitionsBuilder(Opcode: G_CTPOP);
279 if (ST.hasStdExtZbb()) {
280 CTPOPActions.legalFor(Types: {{sXLen, sXLen}})
281 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
282 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
283 } else {
284 CTPOPActions.widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8)
285 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sXLen)
286 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0)
287 .lower();
288 }
289
290 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
291 .legalFor(Types: {p0})
292 .legalFor(Pred: !ST.is64Bit(), Types: {s32})
293 .customFor(Pred: ST.is64Bit(), Types: {s64})
294 .widenScalarToNextPow2(TypeIdx: 0)
295 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
296
297 // TODO: transform illegal vector types into legal vector type
298 getActionDefinitionsBuilder(Opcode: G_FREEZE)
299 .legalFor(Types: {s16, s32, p0})
300 .legalFor(Pred: ST.is64Bit(), Types: {s64})
301 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
302 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
303 .widenScalarToNextPow2(TypeIdx: 0)
304 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen);
305
306 // TODO: transform illegal vector types into legal vector type
307 // TODO: Merge with G_FREEZE?
308 getActionDefinitionsBuilder(
309 Opcodes: {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})
310 .legalFor(Types: {s32, sXLen, p0})
311 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
312 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
313 .widenScalarToNextPow2(TypeIdx: 0)
314 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
315
316 getActionDefinitionsBuilder(Opcode: G_ICMP)
317 .legalFor(Types: {{sXLen, sXLen}, {sXLen, p0}})
318 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
319 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
320 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, MinSize: 8)
321 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
322 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
323
324 getActionDefinitionsBuilder(Opcode: G_SELECT)
325 .legalFor(Types: {{s32, sXLen}, {p0, sXLen}})
326 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
327 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
328 .legalFor(Pred: XLen == 64 || ST.hasStdExtD(), Types: {{s64, sXLen}})
329 .widenScalarToNextPow2(TypeIdx: 0)
330 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)
331 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
332
333 auto &LoadActions = getActionDefinitionsBuilder(Opcode: G_LOAD);
334 auto &StoreActions = getActionDefinitionsBuilder(Opcode: G_STORE);
335 auto &ExtLoadActions = getActionDefinitionsBuilder(Opcodes: {G_SEXTLOAD, G_ZEXTLOAD});
336
337 // Return the alignment needed for scalar memory ops. If unaligned scalar mem
338 // is supported, we only require byte alignment. Otherwise, we need the memory
339 // op to be natively aligned.
340 auto getScalarMemAlign = [&ST](unsigned Size) {
341 return ST.enableUnalignedScalarMem() ? 8 : Size;
342 };
343
344 LoadActions.legalForTypesWithMemDesc(
345 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
346 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
347 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
348 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
349 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
350 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
351 StoreActions.legalForTypesWithMemDesc(
352 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
353 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
354 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
355 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
356 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
357 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
358 ExtLoadActions.legalForTypesWithMemDesc(
359 TypesAndMemDesc: {{.Type0: sXLen, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
360 {.Type0: sXLen, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)}});
361 if (XLen == 64) {
362 LoadActions.legalForTypesWithMemDesc(
363 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
364 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
365 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
366 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
367 StoreActions.legalForTypesWithMemDesc(
368 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
369 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
370 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
371 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
372 ExtLoadActions.legalForTypesWithMemDesc(
373 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)}});
374 } else if (ST.hasStdExtD()) {
375 LoadActions.legalForTypesWithMemDesc(
376 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
377 StoreActions.legalForTypesWithMemDesc(
378 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
379 }
380
381 // Vector loads/stores.
382 if (ST.hasVInstructions()) {
383 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
384 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
385 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
386 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
387 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
388 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
389 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
390 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
391 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
392 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
393 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
394 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
395 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
396 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
397 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
398 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
399 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
400 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
401 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
402 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
403 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
404 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
405 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
406 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
407 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
408 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
409 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
410 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
411 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
412 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
413
414 if (ST.getELen() == 64) {
415 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
416 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
417 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
418 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
419 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
420 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
421 }
422
423 if (ST.hasVInstructionsI64()) {
424 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
425 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
426 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
427 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
428 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
429 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
430 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
431 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
432 }
433
434 // we will take the custom lowering logic if we have scalable vector types
435 // with non-standard alignments
436 LoadActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
437 StoreActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
438
439 // Pointers require that XLen sized elements are legal.
440 if (XLen <= ST.getELen()) {
441 LoadActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
442 StoreActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
443 }
444 }
445
446 LoadActions.widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
447 .lowerIfMemSizeNotByteSizePow2()
448 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
449 .lower();
450 StoreActions
451 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
452 .lowerIfMemSizeNotByteSizePow2()
453 .lower();
454
455 ExtLoadActions.widenScalarToNextPow2(TypeIdx: 0).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).lower();
456
457 getActionDefinitionsBuilder(Opcodes: {G_PTR_ADD, G_PTRMASK}).legalFor(Types: {{p0, sXLen}});
458
459 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
460 .legalFor(Types: {{sXLen, p0}})
461 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
462
463 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
464 .legalFor(Types: {{p0, sXLen}})
465 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
466
467 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
468
469 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {sXLen}).minScalar(TypeIdx: 0, Ty: sXLen);
470
471 getActionDefinitionsBuilder(Opcode: G_BRJT).customFor(Types: {{p0, sXLen}});
472
473 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
474
475 getActionDefinitionsBuilder(Opcode: G_PHI)
476 .legalFor(Types: {p0, s32, sXLen})
477 .widenScalarToNextPow2(TypeIdx: 0)
478 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
479
480 getActionDefinitionsBuilder(Opcodes: {G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
481 .legalFor(Types: {p0});
482
483 if (ST.hasStdExtZmmul()) {
484 getActionDefinitionsBuilder(Opcode: G_MUL)
485 .legalFor(Types: {sXLen})
486 .widenScalarToNextPow2(TypeIdx: 0)
487 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
488
489 // clang-format off
490 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
491 .legalFor(Types: {sXLen})
492 .lower();
493 // clang-format on
494
495 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO}).minScalar(TypeIdx: 0, Ty: sXLen).lower();
496 } else {
497 getActionDefinitionsBuilder(Opcode: G_MUL)
498 .libcallFor(Types: {sXLen, sDoubleXLen})
499 .widenScalarToNextPow2(TypeIdx: 0)
500 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen);
501
502 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).lowerFor(Types: {sXLen});
503
504 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
505 .minScalar(TypeIdx: 0, Ty: sXLen)
506 // Widen sXLen to sDoubleXLen so we can use a single libcall to get
507 // the low bits for the mul result and high bits to do the overflow
508 // check.
509 .widenScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: sXLen),
510 Mutation: LegalizeMutations::changeTo(TypeIdx: 0, Ty: sDoubleXLen))
511 .lower();
512 }
513
514 if (ST.hasStdExtM()) {
515 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV, G_UREM})
516 .legalFor(Types: {sXLen})
517 .customFor(Types: {s32})
518 .libcallFor(Types: {sDoubleXLen})
519 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sDoubleXLen)
520 .widenScalarToNextPow2(TypeIdx: 0);
521 getActionDefinitionsBuilder(Opcode: G_SREM)
522 .legalFor(Types: {sXLen})
523 .libcallFor(Types: {sDoubleXLen})
524 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
525 .widenScalarToNextPow2(TypeIdx: 0);
526 } else {
527 getActionDefinitionsBuilder(Opcodes: {G_UDIV, G_SDIV, G_UREM, G_SREM})
528 .libcallFor(Types: {sXLen, sDoubleXLen})
529 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
530 .widenScalarToNextPow2(TypeIdx: 0);
531 }
532
533 // TODO: Use libcall for sDoubleXLen.
534 getActionDefinitionsBuilder(Opcodes: {G_SDIVREM, G_UDIVREM}).lower();
535
536 getActionDefinitionsBuilder(Opcode: G_ABS)
537 .customFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
538 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
539 .lower();
540
541 getActionDefinitionsBuilder(Opcodes: {G_ABDS, G_ABDU})
542 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
543 .lower();
544
545 getActionDefinitionsBuilder(Opcodes: {G_UMAX, G_UMIN, G_SMAX, G_SMIN})
546 .legalFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
547 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
548 .lower();
549
550 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
551
552 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
553
554 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
555
556 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
557
558 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
559 .lower();
560
561 // On RV64 the 64-bit counter CSRs (cycle/time) are read directly. On RV32
562 // they are custom-legally lowered to a re-read-the-high-half loop (see
563 // legalizeReadCounter).
564 getActionDefinitionsBuilder(Opcodes: {G_READCYCLECOUNTER, G_READSTEADYCOUNTER})
565 .legalFor(Pred: ST.is64Bit(), Types: {s64})
566 .customFor(Pred: !ST.is64Bit(), Types: {s64});
567
568 // FP Operations
569
570 // FIXME: Support s128 for rv32 when libcall handling is able to use sret.
571 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,
572 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,
573 G_FMINIMUMNUM})
574 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
575 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
576 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
577 .libcallFor(Types: {s32, s64})
578 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
579
580 getActionDefinitionsBuilder(Opcodes: {G_FNEG, G_FABS})
581 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
582 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
583 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
584 .lowerFor(Types: {s32, s64, s128});
585
586 getActionDefinitionsBuilder(Opcode: G_FREM)
587 .libcallFor(Types: {s32, s64})
588 .libcallFor(Pred: ST.is64Bit(), Types: {s128})
589 .minScalar(TypeIdx: 0, Ty: s32)
590 .scalarize(TypeIdx: 0);
591
592 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
593 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, s32}})
594 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s64}, {s32, s64}, {s64, s32}})
595 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s16}, {s16, s32}, {s32, s16}})
596 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}, {s64, s16}})
597 .lower();
598
599 // FIXME: Use Zfhmin.
600 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
601 .legalFor(Pred: ST.hasStdExtD(), Types: {{s32, s64}})
602 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s32}})
603 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}})
604 .libcallFor(Types: {{s32, s64}})
605 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}});
606 getActionDefinitionsBuilder(Opcode: G_FPEXT)
607 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s32}})
608 .legalFor(Pred: ST.hasStdExtZfhmin(), Types: {{s32, s16}})
609 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s64, s16}})
610 .libcallFor(Pred: !ST.hasStdExtZfhmin(), Types: {{s32, s16}})
611 .libcallFor(Types: {{s64, s32}})
612 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}});
613
614 getActionDefinitionsBuilder(Opcode: G_FCMP)
615 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
616 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
617 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
618 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
619 .libcallFor(Types: {{sXLen, s32}, {sXLen, s64}})
620 .libcallFor(Pred: ST.is64Bit(), Types: {{sXLen, s128}});
621
622 // TODO: Support vector version of G_IS_FPCLASS.
623 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
624 .customFor(Pred: ST.hasStdExtF(), Types: {{s1, s32}})
625 .customFor(Pred: ST.hasStdExtD(), Types: {{s1, s64}})
626 .customFor(Pred: ST.hasStdExtZfh(), Types: {{s1, s16}})
627 .lower();
628
629 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
630 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
631 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
632 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
633 .customFor(Pred: !ST.is64Bit(), Types: {s32})
634 .customFor(Pred: ST.is64Bit(), Types: {s32, s64})
635 .lowerFor(Types: {s64, s128});
636
637 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
638 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
639 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
640 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
641 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
642 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
643 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
644 .widenScalarToNextPow2(TypeIdx: 0)
645 .minScalar(TypeIdx: 0, Ty: s32)
646 // The magnitude of a half is at most 65504, so with Zfh use fcvt.w[u].h
647 // and extend the i32 result. Otherwise promote the half source to float
648 // (via fcvt.s.h with Zfhmin, __extendhfsf2 without) and use the float
649 // conversion. On RV32, exclude i64 results here so that they get
650 // narrowed to i32 first.
651 .widenScalarIf(
652 Predicate: [=, &ST](const LegalityQuery &Query) {
653 return Query.Types[1] == f16 && !ST.hasStdExtZfh() &&
654 (ST.is64Bit() || Query.Types[0] == s32);
655 },
656 Mutation: changeTo(TypeIdx: 1, Ty: s32))
657 .libcallFor(Pred: !ST.hasStdExtZfhmin(), Types: {{s64, f16}})
658 .narrowScalarFor(Types: {{s64, f16}}, Mutation: changeTo(TypeIdx: 0, Ty: s32))
659 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
660 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}}) // FIXME RV32.
661 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}, {s128, s128}});
662
663 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
664 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
665 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
666 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
667 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
668 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
669 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
670 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16), Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: s32))
671 .libcallFor(Types: {{s32, s32},
672 {s64, s32},
673 {s32, s64},
674 {s64, s64},
675 {s32, s128},
676 {s64, s128}});
677
678 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
679 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
680 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
681 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
682 .minScalar(TypeIdx: 0, Ty: sXLen)
683 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16), Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: s32))
684 .libcallFor(Types: {{s32, s32},
685 {s64, s32},
686 {s32, s64},
687 {s64, s64},
688 {s32, s128},
689 {s64, s128}});
690
691 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
692 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, sXLen}})
693 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, sXLen}})
694 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, sXLen}})
695 .widenScalarToNextPow2(TypeIdx: 1)
696 // Promote to XLen if the operation is legal.
697 .widenScalarIf(
698 Predicate: [=, &ST](const LegalityQuery &Query) {
699 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
700 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
701 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
702 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
703 (ST.hasStdExtZfh() &&
704 Query.Types[0].getSizeInBits() == 16));
705 },
706 Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: sXLen))
707 // Otherwise only promote to s32 since we have si libcalls.
708 .minScalar(TypeIdx: 1, Ty: s32)
709 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
710 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}}) // FIXME RV32.
711 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}, {s128, s128}});
712
713 // FIXME: We can do custom inline expansion like SelectionDAG.
714 getActionDefinitionsBuilder(Opcodes: {G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
715 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
716 G_INTRINSIC_ROUNDEVEN})
717 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
718 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
719 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16})
720 .libcallFor(Types: {s32, s64})
721 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
722
723 getActionDefinitionsBuilder(Opcodes: {G_FMAXIMUM, G_FMINIMUM})
724 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
725 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
726 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16});
727
728 getActionDefinitionsBuilder(Opcodes: {G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
729 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
730 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
731 G_FTANH, G_FMODF})
732 .libcallFor(Types: {s32, s64})
733 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
734 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
735 .libcallFor(Types: {{s32, s32}, {s64, s32}})
736 .libcallFor(Pred: ST.is64Bit(), Types: {s128, s32});
737
738 getActionDefinitionsBuilder(Opcode: G_FCANONICALIZE)
739 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
740 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
741 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16});
742
743 getActionDefinitionsBuilder(Opcode: G_VASTART).customFor(Types: {p0});
744
745 // va_list must be a pointer, but most sized types are pretty easy to handle
746 // as the destination.
747 getActionDefinitionsBuilder(Opcode: G_VAARG)
748 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types
749 // other than sXLen.
750 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
751 .lowerForCartesianProduct(Types0: {sXLen, p0}, Types1: {p0});
752
753 getActionDefinitionsBuilder(Opcode: G_VSCALE)
754 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
755 .customFor(Types: {sXLen});
756
757 auto &SplatActions =
758 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
759 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
760 P1: typeIs(TypeIdx: 1, TypesInit: sXLen)))
761 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: typeIs(TypeIdx: 1, TypesInit: s1)));
762 // Handle case of s64 element vectors on RV32. If the subtarget does not have
763 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget
764 // does have f64, then we don't know whether the type is an f64 or an i64,
765 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,
766 // depending on how the instructions it consumes are legalized. They are not
767 // legalized yet since legalization is in reverse postorder, so we cannot
768 // make the decision at this moment.
769 if (XLen == 32) {
770 if (ST.hasVInstructionsF64() && ST.hasStdExtD())
771 SplatActions.legalIf(Predicate: all(
772 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
773 else if (ST.hasVInstructionsI64())
774 SplatActions.customIf(Predicate: all(
775 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
776 }
777
778 SplatActions.clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
779
780 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
781 LLT DstTy = Query.Types[0];
782 LLT SrcTy = Query.Types[1];
783 return DstTy.getElementType() == LLT::scalar(SizeInBits: 1) &&
784 DstTy.getElementCount().getKnownMinValue() >= 8 &&
785 SrcTy.getElementCount().getKnownMinValue() >= 8;
786 };
787 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
788 // We don't have the ability to slide mask vectors down indexed by their
789 // i1 elements; the smallest we can do is i8. Often we are able to bitcast
790 // to equivalent i8 vectors.
791 .bitcastIf(
792 Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
793 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST), args: ExtractSubvecBitcastPred),
794 Mutation: [=](const LegalityQuery &Query) {
795 LLT CastTy = LLT::vector(
796 EC: Query.Types[0].getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8);
797 return std::pair(0, CastTy);
798 })
799 .customIf(Predicate: LegalityPredicates::any(
800 P0: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
801 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)),
802 P1: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
803 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST))));
804
805 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR)
806 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
807 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
808 .customIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
809 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)));
810
811 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
812 .lowerIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
813
814 getActionDefinitionsBuilder(Opcodes: {G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
815 G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
816 G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
817 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
818 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
819 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
820
821 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_SUB)
822 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
823 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
824 .lower();
825
826 getActionDefinitionsBuilder(
827 Opcodes: {G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
828 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
829 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
830 .unsupported();
831
832 getActionDefinitionsBuilder(Opcode: G_PREFETCH).legalIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p0));
833
834 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
835 LLT VecTy = Query.Types[0];
836 LLT EltTy = Query.Types[1];
837 return VecTy.getElementType() == EltTy;
838 };
839
840 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
841 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
842 P1: InsertVectorEltPred, args: typeIs(TypeIdx: 2, TypesInit: sXLen)))
843 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: InsertVectorEltPred,
844 args: typeIs(TypeIdx: 2, TypesInit: sXLen)));
845
846 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
847 .alwaysLegal();
848
849 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
850
851 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
852
853 verify(MII: *ST.getInstrInfo());
854}
855
856bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
857 MachineInstr &MI) const {
858 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
859
860 if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
861 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {
862 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {
863 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
864 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
865
866 auto OldScalar = MI.getOperand(i: II->ScalarOperand + 2).getReg();
867 // Legalize integer vx form intrinsic.
868 if (MRI.getType(Reg: OldScalar).isScalar()) {
869 if (MRI.getType(Reg: OldScalar).getSizeInBits() < sXLen.getSizeInBits()) {
870 Helper.Observer.changingInstr(MI);
871 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: II->ScalarOperand + 2,
872 ExtOpcode: TargetOpcode::G_ANYEXT);
873 Helper.Observer.changedInstr(MI);
874 } else if (MRI.getType(Reg: OldScalar).getSizeInBits() >
875 sXLen.getSizeInBits()) {
876 // TODO: i64 in riscv32.
877 return false;
878 }
879 }
880 }
881 return true;
882 }
883
884 switch (IntrinsicID) {
885 default:
886 return false;
887 case Intrinsic::riscv_clmulh:
888 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULH, DstOps: {MI.getOperand(i: 0)},
889 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
890 MI.eraseFromParent();
891 return true;
892 case Intrinsic::riscv_clmulr:
893 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULR, DstOps: {MI.getOperand(i: 0)},
894 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
895 MI.eraseFromParent();
896 return true;
897 case Intrinsic::vacopy: {
898 // vacopy arguments must be legal because of the intrinsic signature.
899 // No need to check here.
900
901 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
902 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
903 MachineFunction &MF = *MI.getMF();
904 const DataLayout &DL = MIRBuilder.getDataLayout();
905 LLVMContext &Ctx = MF.getFunction().getContext();
906
907 Register DstLst = MI.getOperand(i: 1).getReg();
908 LLT PtrTy = MRI.getType(Reg: DstLst);
909
910 // Load the source va_list
911 Align Alignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
912 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
913 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: Alignment);
914 auto Tmp = MIRBuilder.buildLoad(Res: PtrTy, Addr: MI.getOperand(i: 2), MMO&: *LoadMMO);
915
916 // Store the result in the destination va_list
917 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
918 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: Alignment);
919 MIRBuilder.buildStore(Val: Tmp, Addr: DstLst, MMO&: *StoreMMO);
920
921 MI.eraseFromParent();
922 return true;
923 }
924 case Intrinsic::riscv_vsetvli:
925 case Intrinsic::riscv_vsetvlimax:
926 case Intrinsic::riscv_masked_atomicrmw_add:
927 case Intrinsic::riscv_masked_atomicrmw_sub:
928 case Intrinsic::riscv_masked_atomicrmw_xchg:
929 case Intrinsic::riscv_masked_atomicrmw_max:
930 case Intrinsic::riscv_masked_atomicrmw_min:
931 case Intrinsic::riscv_masked_atomicrmw_umax:
932 case Intrinsic::riscv_masked_atomicrmw_umin:
933 case Intrinsic::riscv_masked_cmpxchg:
934 return true;
935 }
936}
937
938bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
939 MachineIRBuilder &MIRBuilder) const {
940 // Stores the address of the VarArgsFrameIndex slot into the memory location
941 assert(MI.getOpcode() == TargetOpcode::G_VASTART);
942 MachineFunction *MF = MI.getParent()->getParent();
943 RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>();
944 int FI = FuncInfo->getVarArgsFrameIndex();
945 LLT AddrTy = MIRBuilder.getMRI()->getType(Reg: MI.getOperand(i: 0).getReg());
946 auto FINAddr = MIRBuilder.buildFrameIndex(Res: AddrTy, Idx: FI);
947 assert(MI.hasOneMemOperand());
948 MIRBuilder.buildStore(Val: FINAddr, Addr: MI.getOperand(i: 0).getReg(),
949 MMO&: *MI.memoperands()[0]);
950 MI.eraseFromParent();
951 return true;
952}
953
954bool RISCVLegalizerInfo::legalizeReadCounter(
955 MachineInstr &MI, MachineIRBuilder &MIRBuilder,
956 GISelChangeObserver &Observer) const {
957 assert((MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER ||
958 MI.getOpcode() == TargetOpcode::G_READSTEADYCOUNTER) &&
959 "Unexpected opcode");
960 assert(!STI.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
961 "has custom type legalization on riscv32");
962
963 // On RV32 a 64-bit counter CSR must be read as two 32-bit halves. Because
964 // the count may wrap between the two reads, re-read the high half and loop
965 // until the two high reads agree.
966 int64_t LoCounter, HiCounter;
967 if (MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER) {
968 LoCounter = RISCVSysReg::cycle;
969 HiCounter = RISCVSysReg::cycleh;
970 } else {
971 LoCounter = RISCVSysReg::time;
972 HiCounter = RISCVSysReg::timeh;
973 }
974
975 MachineBasicBlock *BB = MI.getParent();
976 MachineFunction &MF = *BB->getParent();
977 const BasicBlock *LLVMBB = BB->getBasicBlock();
978 DebugLoc DL = MI.getDebugLoc();
979 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
980
981 // Split BB into an entry that falls through into a loop block, and a done
982 // block that receives the remainder of BB and its original successors.
983 MachineFunction::iterator It = std::next(x: BB->getIterator());
984 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
985 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
986 MF.insert(MBBI: It, MBB: LoopMBB);
987 MF.insert(MBBI: It, MBB: DoneMBB);
988
989 // Splice the instructions after the readcyclecounter into DoneMBB, notifying
990 // the observer about each moved instruction so CSEInfo stays consistent.
991 for (MachineBasicBlock::iterator I = std::next(x: MI.getIterator()),
992 E = BB->end();
993 I != E; ++I)
994 Observer.changingInstr(MI&: *I);
995 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
996 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
997 for (MachineInstr &MovedMI : DoneMBB->instrs())
998 Observer.changedInstr(MI&: MovedMI);
999 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
1000 BB->addSuccessor(Succ: LoopMBB);
1001
1002 LLT S32 = LLT::scalar(SizeInBits: 32);
1003 // Generic vregs carry the s32 type for G_MERGE_VALUES below, but are also
1004 // constrained to GPR so the target CSRRS/BNE instructions satisfy the
1005 // verifier's register-class constraints.
1006 auto CreateGPR = [&]() {
1007 Register R = MRI.createGenericVirtualRegister(Ty: S32);
1008 MRI.setRegClass(Reg: R, RC: &RISCV::GPRRegClass);
1009 return R;
1010 };
1011 Register LoReg = CreateGPR();
1012 Register HiReg = CreateGPR();
1013 Register ReadAgainReg = CreateGPR();
1014
1015 // read:
1016 // csrrs HiReg, counterh # high word
1017 // csrrs LoReg, counter # low word
1018 // csrrs ReadAgainReg, counterh
1019 // bne HiReg, ReadAgainReg, read
1020 // Emit the target instructions directly with BuildMI.
1021 const RISCVInstrInfo *TII = STI.getInstrInfo();
1022 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
1023 .addImm(Val: HiCounter)
1024 .addReg(RegNo: RISCV::X0);
1025 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
1026 .addImm(Val: LoCounter)
1027 .addReg(RegNo: RISCV::X0);
1028 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
1029 .addImm(Val: HiCounter)
1030 .addReg(RegNo: RISCV::X0);
1031
1032 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
1033 .addReg(RegNo: HiReg)
1034 .addReg(RegNo: ReadAgainReg)
1035 .addMBB(MBB: LoopMBB);
1036
1037 LoopMBB->addSuccessor(Succ: LoopMBB);
1038 LoopMBB->addSuccessor(Succ: DoneMBB);
1039
1040 // Re-pair the two halves into the 64-bit result.
1041 Register DstReg = MI.getOperand(i: 0).getReg();
1042 Observer.erasingInstr(MI);
1043 MI.eraseFromParent();
1044
1045 MIRBuilder.setInsertPt(MBB&: *DoneMBB, II: DoneMBB->begin());
1046 MIRBuilder.setDebugLoc(DL);
1047 MIRBuilder.buildMergeValues(Res: DstReg, Ops: {LoReg, HiReg});
1048 return true;
1049}
1050
1051bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,
1052 MachineIRBuilder &MIRBuilder) const {
1053 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1054 auto &MF = *MI.getParent()->getParent();
1055 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
1056 unsigned EntrySize = MJTI->getEntrySize(TD: MF.getDataLayout());
1057
1058 Register PtrReg = MI.getOperand(i: 0).getReg();
1059 LLT PtrTy = MRI.getType(Reg: PtrReg);
1060 Register IndexReg = MI.getOperand(i: 2).getReg();
1061 LLT IndexTy = MRI.getType(Reg: IndexReg);
1062
1063 if (!isPowerOf2_32(Value: EntrySize))
1064 return false;
1065
1066 auto ShiftAmt = MIRBuilder.buildConstant(Res: IndexTy, Val: Log2_32(Value: EntrySize));
1067 IndexReg = MIRBuilder.buildShl(Dst: IndexTy, Src0: IndexReg, Src1: ShiftAmt).getReg(Idx: 0);
1068
1069 auto Addr = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: IndexReg);
1070
1071 MachineMemOperand *MMO = MF.getMachineMemOperand(
1072 PtrInfo: MachinePointerInfo::getJumpTable(MF), F: MachineMemOperand::MOLoad,
1073 Size: EntrySize, BaseAlignment: Align(MJTI->getEntryAlignment(TD: MF.getDataLayout())));
1074
1075 Register TargetReg;
1076 switch (MJTI->getEntryKind()) {
1077 default:
1078 return false;
1079 case MachineJumpTableInfo::EK_LabelDifference32: {
1080 // For PIC, the sequence is:
1081 // BRIND(load(Jumptable + index) + RelocBase)
1082 // RelocBase can be JumpTable, GOT or some sort of global base.
1083 unsigned LoadOpc =
1084 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;
1085 auto Load = MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: IndexTy, Addr, MMO&: *MMO);
1086 TargetReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: Load).getReg(Idx: 0);
1087 break;
1088 }
1089 case MachineJumpTableInfo::EK_Custom32: {
1090 auto Load = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: IndexTy,
1091 Addr, MMO&: *MMO);
1092 TargetReg = MIRBuilder.buildIntToPtr(Dst: PtrTy, Src: Load).getReg(Idx: 0);
1093 break;
1094 }
1095 case MachineJumpTableInfo::EK_BlockAddress:
1096 TargetReg = MIRBuilder.buildLoad(Res: PtrTy, Addr, MMO&: *MMO).getReg(Idx: 0);
1097 break;
1098 }
1099
1100 MIRBuilder.buildBrIndirect(Tgt: TargetReg);
1101
1102 MI.eraseFromParent();
1103 return true;
1104}
1105
1106bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,
1107 bool ShouldOptForSize) const {
1108 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);
1109 int64_t Imm = APImm.getSExtValue();
1110 // All simm32 constants should be handled by isel.
1111 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
1112 // this check redundant, but small immediates are common so this check
1113 // should have better compile time.
1114 if (isInt<32>(x: Imm))
1115 return false;
1116
1117 // We only need to cost the immediate, if constant pool lowering is enabled.
1118 if (!STI.useConstantPoolForLargeInts())
1119 return false;
1120
1121 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI);
1122 if (Seq.size() <= STI.getMaxBuildIntsCost())
1123 return false;
1124
1125 // Optimizations below are disabled for opt size. If we're optimizing for
1126 // size, use a constant pool.
1127 if (ShouldOptForSize)
1128 return true;
1129 //
1130 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
1131 // that if it will avoid a constant pool.
1132 // It will require an extra temporary register though.
1133 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
1134 // low and high 32 bits are the same and bit 31 and 63 are set.
1135 unsigned ShiftAmt, AddOpc;
1136 RISCVMatInt::InstSeq SeqLo =
1137 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI, ShiftAmt, AddOpc);
1138 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
1139}
1140
1141bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
1142 MachineIRBuilder &MIB) const {
1143 Register Dst = MI.getOperand(i: 0).getReg();
1144
1145 // We define our scalable vector types for lmul=1 to use a 64 bit known
1146 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
1147 // vscale as VLENB / 8.
1148 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
1149 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1150 // Support for VLEN==32 is incomplete.
1151 return false;
1152
1153 // We assume VLENB is a multiple of 8. We manually choose the best shift
1154 // here because SimplifyDemandedBits isn't always able to simplify it.
1155 uint64_t Val = MI.getOperand(i: 1).getCImm()->getZExtValue();
1156 if (isPowerOf2_64(Value: Val)) {
1157 uint64_t Log2 = Log2_64(Value: Val);
1158 if (Log2 < 3) {
1159 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1160 MIB.buildLShr(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3 - Log2),
1161 Flags: MachineInstr::IsExact);
1162 } else if (Log2 > 3) {
1163 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1164 MIB.buildShl(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Log2 - 3));
1165 } else {
1166 MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {Dst}, SrcOps: {});
1167 }
1168 } else if ((Val % 8) == 0) {
1169 // If the multiplier is a multiple of 8, scale it down to avoid needing
1170 // to shift the VLENB value.
1171 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1172 MIB.buildMul(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Val / 8));
1173 } else {
1174 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1175 auto VScale = MIB.buildLShr(Dst: sXLen, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3),
1176 Flags: MachineInstr::IsExact);
1177 MIB.buildMul(Dst, Src0: VScale, Src1: MIB.buildConstant(Res: sXLen, Val));
1178 }
1179 MI.eraseFromParent();
1180 return true;
1181}
1182
1183// Custom-lower extensions from mask vectors by using a vselect either with 1
1184// for zero/any-extension or -1 for sign-extension:
1185// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
1186// Note that any-extension is lowered identically to zero-extension.
1187bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,
1188 MachineIRBuilder &MIB) const {
1189
1190 unsigned Opc = MI.getOpcode();
1191 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||
1192 Opc == TargetOpcode::G_ANYEXT);
1193
1194 MachineRegisterInfo &MRI = *MIB.getMRI();
1195 Register Dst = MI.getOperand(i: 0).getReg();
1196 Register Src = MI.getOperand(i: 1).getReg();
1197
1198 LLT DstTy = MRI.getType(Reg: Dst);
1199 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;
1200 LLT DstEltTy = DstTy.getElementType();
1201 auto SplatZero = MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: 0));
1202 auto SplatTrue =
1203 MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: ExtTrueVal));
1204 MIB.buildSelect(Res: Dst, Tst: Src, Op0: SplatTrue, Op1: SplatZero);
1205
1206 MI.eraseFromParent();
1207 return true;
1208}
1209
1210bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,
1211 LegalizerHelper &Helper,
1212 MachineIRBuilder &MIB) const {
1213 assert((isa<GLoad>(MI) || isa<GStore>(MI)) &&
1214 "Machine instructions must be Load/Store.");
1215 MachineRegisterInfo &MRI = *MIB.getMRI();
1216 MachineFunction *MF = MI.getMF();
1217 const DataLayout &DL = MIB.getDataLayout();
1218 LLVMContext &Ctx = MF->getFunction().getContext();
1219
1220 Register DstReg = MI.getOperand(i: 0).getReg();
1221 LLT DataTy = MRI.getType(Reg: DstReg);
1222 if (!DataTy.isVector())
1223 return false;
1224
1225 if (!MI.hasOneMemOperand())
1226 return false;
1227
1228 MachineMemOperand *MMO = *MI.memoperands_begin();
1229
1230 const auto *TLI = STI.getTargetLowering();
1231 EVT VT = EVT::getEVT(Ty: getTypeForLLT(Ty: DataTy, C&: Ctx));
1232
1233 if (TLI->allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT, MMO: *MMO))
1234 return true;
1235
1236 unsigned EltSizeBits = DataTy.getScalarSizeInBits();
1237 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
1238 "Unexpected unaligned RVV load type");
1239
1240 // Calculate the new vector type with i8 elements
1241 unsigned NumElements =
1242 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);
1243 LLT NewDataTy = LLT::scalable_vector(MinNumElements: NumElements, ScalarSizeInBits: 8);
1244
1245 Helper.bitcast(MI, TypeIdx: 0, Ty: NewDataTy);
1246
1247 return true;
1248}
1249
1250/// Return the type of the mask type suitable for masking the provided
1251/// vector type. This is simply an i1 element type vector of the same
1252/// (possibly scalable) length.
1253static LLT getMaskTypeFor(LLT VecTy) {
1254 assert(VecTy.isVector());
1255 ElementCount EC = VecTy.getElementCount();
1256 return LLT::vector(EC, ScalarTy: LLT::scalar(SizeInBits: 1));
1257}
1258
1259/// Creates an all ones mask suitable for masking a vector of type VecTy with
1260/// vector length VL.
1261static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,
1262 MachineIRBuilder &MIB,
1263 MachineRegisterInfo &MRI) {
1264 LLT MaskTy = getMaskTypeFor(VecTy);
1265 return MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {MaskTy}, SrcOps: {VL});
1266}
1267
1268/// Gets the two common "VL" operands: an all-ones mask and the vector length.
1269/// VecTy is a scalable vector type.
1270static std::pair<MachineInstrBuilder, MachineInstrBuilder>
1271buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1272 assert(VecTy.isScalableVector() && "Expecting scalable container type");
1273 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
1274 LLT XLenTy(STI.getXLenVT());
1275 auto VL = MIB.buildConstant(Res: XLenTy, Val: -1);
1276 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
1277 return {Mask, VL};
1278}
1279
1280static MachineInstrBuilder
1281buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
1282 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
1283 MachineRegisterInfo &MRI) {
1284 // TODO: If the Hi bits of the splat are undefined, then it's fine to just
1285 // splat Lo even if it might be sign extended. I don't think we have
1286 // introduced a case where we're build a s64 where the upper bits are undef
1287 // yet.
1288
1289 // Fall back to a stack store and stride x0 vector load.
1290 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in
1291 // preprocessDAG in SDAG.
1292 return MIB.buildInstr(Opc: RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, DstOps: {Dst},
1293 SrcOps: {Passthru, Lo, Hi, VL});
1294}
1295
1296static MachineInstrBuilder
1297buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
1298 const SrcOp &Scalar, const SrcOp &VL,
1299 MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1300 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
1301 auto Unmerge = MIB.buildUnmerge(Res: LLT::scalar(SizeInBits: 32), Op: Scalar);
1302 return buildSplatPartsS64WithVL(Dst, Passthru, Lo: Unmerge.getReg(Idx: 0),
1303 Hi: Unmerge.getReg(Idx: 1), VL, MIB, MRI);
1304}
1305
1306// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a
1307// legal equivalently-sized i8 type, so we can use that as a go-between.
1308// Splats of s1 types that have constant value can be legalized as VMSET_VL or
1309// VMCLR_VL.
1310bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
1311 MachineIRBuilder &MIB) const {
1312 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);
1313
1314 MachineRegisterInfo &MRI = *MIB.getMRI();
1315
1316 Register Dst = MI.getOperand(i: 0).getReg();
1317 Register SplatVal = MI.getOperand(i: 1).getReg();
1318
1319 LLT VecTy = MRI.getType(Reg: Dst);
1320 LLT XLenTy(STI.getXLenVT());
1321
1322 // Handle case of s64 element vectors on rv32
1323 if (XLenTy.getSizeInBits() == 32 &&
1324 VecTy.getElementType().getSizeInBits() == 64) {
1325 auto [_, VL] = buildDefaultVLOps(VecTy: MRI.getType(Reg: Dst), MIB, MRI);
1326 buildSplatSplitS64WithVL(Dst, Passthru: MIB.buildUndef(Res: VecTy), Scalar: SplatVal, VL, MIB,
1327 MRI);
1328 MI.eraseFromParent();
1329 return true;
1330 }
1331
1332 // All-zeros or all-ones splats are handled specially.
1333 MachineInstr &SplatValMI = *MRI.getVRegDef(Reg: SplatVal);
1334 if (isAllOnesOrAllOnesSplat(MI: SplatValMI, MRI)) {
1335 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1336 MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {Dst}, SrcOps: {VL});
1337 MI.eraseFromParent();
1338 return true;
1339 }
1340 if (isNullOrNullSplat(MI: SplatValMI, MRI)) {
1341 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1342 MIB.buildInstr(Opc: RISCV::G_VMCLR_VL, DstOps: {Dst}, SrcOps: {VL});
1343 MI.eraseFromParent();
1344 return true;
1345 }
1346
1347 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all
1348 // ones) by promoting it to an s8 splat.
1349 LLT InterEltTy = LLT::scalar(SizeInBits: 8);
1350 LLT InterTy = VecTy.changeElementType(NewEltTy: InterEltTy);
1351 auto ZExtSplatVal = MIB.buildZExt(Res: InterEltTy, Op: SplatVal);
1352 auto And =
1353 MIB.buildAnd(Dst: InterEltTy, Src0: ZExtSplatVal, Src1: MIB.buildConstant(Res: InterEltTy, Val: 1));
1354 auto LHS = MIB.buildSplatVector(Res: InterTy, Val: And);
1355 auto ZeroSplat =
1356 MIB.buildSplatVector(Res: InterTy, Val: MIB.buildConstant(Res: InterEltTy, Val: 0));
1357 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: LHS, Op1: ZeroSplat);
1358 MI.eraseFromParent();
1359 return true;
1360}
1361
1362static LLT getLMUL1Ty(LLT VecTy) {
1363 assert(VecTy.getElementType().getSizeInBits() <= 64 &&
1364 "Unexpected vector LLT");
1365 return LLT::scalable_vector(MinNumElements: RISCV::RVVBitsPerBlock /
1366 VecTy.getElementType().getSizeInBits(),
1367 ScalarTy: VecTy.getElementType());
1368}
1369
1370bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
1371 MachineIRBuilder &MIB) const {
1372 GExtractSubvector &ES = cast<GExtractSubvector>(Val&: MI);
1373
1374 MachineRegisterInfo &MRI = *MIB.getMRI();
1375
1376 Register Dst = ES.getReg(Idx: 0);
1377 Register Src = ES.getSrcVec();
1378 uint64_t Idx = ES.getIndexImm();
1379
1380 // With an index of 0 this is a cast-like subvector, which can be performed
1381 // with subregister operations.
1382 if (Idx == 0)
1383 return true;
1384
1385 LLT LitTy = MRI.getType(Reg: Dst);
1386 LLT BigTy = MRI.getType(Reg: Src);
1387
1388 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1389 // We can't slide this mask vector up indexed by its i1 elements.
1390 // This poses a problem when we wish to insert a scalable vector which
1391 // can't be re-expressed as a larger type. Just choose the slow path and
1392 // extend to a larger type, then truncate back down.
1393 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1394 LLT ExtLitTy = LitTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1395 auto BigZExt = MIB.buildZExt(Res: ExtBigTy, Op: Src);
1396 auto ExtractZExt = MIB.buildExtractSubvector(Res: ExtLitTy, Src: BigZExt, Index: Idx);
1397 auto SplatZero = MIB.buildSplatVector(
1398 Res: ExtLitTy, Val: MIB.buildConstant(Res: ExtLitTy.getElementType(), Val: 0));
1399 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: ExtractZExt, Op1: SplatZero);
1400 MI.eraseFromParent();
1401 return true;
1402 }
1403
1404 // extract_subvector scales the index by vscale if the subvector is scalable,
1405 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
1406 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1407 MVT LitTyMVT = getMVTForLLT(Ty: LitTy);
1408 auto Decompose =
1409 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1410 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: LitTyMVT, InsertExtractIdx: Idx, TRI);
1411 unsigned RemIdx = Decompose.second;
1412
1413 // If the Idx has been completely eliminated then this is a subvector extract
1414 // which naturally aligns to a vector register. These can easily be handled
1415 // using subregister manipulation.
1416 if (RemIdx == 0)
1417 return true;
1418
1419 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
1420 // was > M1 then the index would need to be a multiple of VLMAX, and so would
1421 // divide exactly.
1422 assert(
1423 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||
1424 RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVVType::LMUL_1);
1425
1426 // If the vector type is an LMUL-group type, extract a subvector equal to the
1427 // nearest full vector register type.
1428 LLT InterLitTy = BigTy;
1429 Register Vec = Src;
1430 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1431 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1432 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
1433 // we should have successfully decomposed the extract into a subregister.
1434 assert(Decompose.first != RISCV::NoSubRegister);
1435 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1436 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
1437 // specified on the source Register (the equivalent) since generic virtual
1438 // register does not allow subregister index.
1439 Vec = MIB.buildExtractSubvector(Res: InterLitTy, Src, Index: Idx - RemIdx).getReg(Idx: 0);
1440 }
1441
1442 // Slide this vector register down by the desired number of elements in order
1443 // to place the desired subvector starting at element 0.
1444 const LLT XLenTy(STI.getXLenVT());
1445 auto SlidedownAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1446 auto [Mask, VL] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1447 uint64_t Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
1448 auto Slidedown = MIB.buildInstr(
1449 Opc: RISCV::G_VSLIDEDOWN_VL, DstOps: {InterLitTy},
1450 SrcOps: {MIB.buildUndef(Res: InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
1451
1452 // Now the vector is in the right position, extract our final subvector. This
1453 // should resolve to a COPY.
1454 MIB.buildExtractSubvector(Res: Dst, Src: Slidedown, Index: 0);
1455
1456 MI.eraseFromParent();
1457 return true;
1458}
1459
1460bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,
1461 LegalizerHelper &Helper,
1462 MachineIRBuilder &MIB) const {
1463 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
1464
1465 MachineRegisterInfo &MRI = *MIB.getMRI();
1466
1467 Register Dst = IS.getReg(Idx: 0);
1468 Register BigVec = IS.getBigVec();
1469 Register LitVec = IS.getSubVec();
1470 uint64_t Idx = IS.getIndexImm();
1471
1472 LLT BigTy = MRI.getType(Reg: BigVec);
1473 LLT LitTy = MRI.getType(Reg: LitVec);
1474
1475 if (Idx == 0 && mi_match(R: BigVec, MRI, P: m_GImplicitDef()))
1476 return true;
1477
1478 // We don't have the ability to slide mask vectors up indexed by their i1
1479 // elements; the smallest we can do is i8. Often we are able to bitcast to
1480 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i8
1481 // vectors and truncate down after the insert.
1482 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1483 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();
1484 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();
1485 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)
1486 return Helper.bitcast(
1487 MI&: IS, TypeIdx: 0,
1488 Ty: LLT::vector(EC: BigTy.getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8));
1489
1490 // We can't slide this mask vector up indexed by its i1 elements.
1491 // This poses a problem when we wish to insert a scalable vector which
1492 // can't be re-expressed as a larger type. Just choose the slow path and
1493 // extend to a larger type, then truncate back down.
1494 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1495 return Helper.widenScalar(MI&: IS, TypeIdx: 0, WideTy: ExtBigTy);
1496 }
1497
1498 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1499 unsigned SubRegIdx, RemIdx;
1500 std::tie(args&: SubRegIdx, args&: RemIdx) =
1501 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1502 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: getMVTForLLT(Ty: LitTy), InsertExtractIdx: Idx, TRI);
1503
1504 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
1505 assert(isPowerOf2_64(
1506 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));
1507 bool ExactlyVecRegSized =
1508 STI.expandVScale(X: LitTy.getSizeInBits())
1509 .isKnownMultipleOf(RHS: STI.expandVScale(X: VecRegSize));
1510
1511 // If the Idx has been completely eliminated and this subvector's size is a
1512 // vector register or a multiple thereof, or the surrounding elements are
1513 // undef, then this is a subvector insert which naturally aligns to a vector
1514 // register. These can easily be handled using subregister manipulation.
1515 if (RemIdx == 0 && ExactlyVecRegSized)
1516 return true;
1517
1518 // If the subvector is smaller than a vector register, then the insertion
1519 // must preserve the undisturbed elements of the register. We do this by
1520 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
1521 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
1522 // subvector within the vector register, and an INSERT_SUBVECTOR of that
1523 // LMUL=1 type back into the larger vector (resolving to another subregister
1524 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
1525 // to avoid allocating a large register group to hold our subvector.
1526
1527 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
1528 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
1529 // (in our case undisturbed). This means we can set up a subvector insertion
1530 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
1531 // size of the subvector.
1532 const LLT XLenTy(STI.getXLenVT());
1533 LLT InterLitTy = BigTy;
1534 Register AlignedExtract = BigVec;
1535 unsigned AlignedIdx = Idx - RemIdx;
1536 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1537 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1538 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1539 // Extract a subvector equal to the nearest full vector register type. This
1540 // should resolve to a G_EXTRACT on a subreg.
1541 AlignedExtract =
1542 MIB.buildExtractSubvector(Res: InterLitTy, Src: BigVec, Index: AlignedIdx).getReg(Idx: 0);
1543 }
1544
1545 auto Insert = MIB.buildInsertSubvector(Res: InterLitTy, Src0: MIB.buildUndef(Res: InterLitTy),
1546 Src1: LitVec, Index: 0);
1547
1548 auto [Mask, _] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1549 auto VL = MIB.buildVScale(Res: XLenTy, MinElts: LitTy.getElementCount().getKnownMinValue());
1550
1551 // If we're inserting into the lowest elements, use a tail undisturbed
1552 // vmv.v.v.
1553 MachineInstrBuilder Inserted;
1554 bool NeedInsertSubvec =
1555 TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(), RHS: InterLitTy.getSizeInBits());
1556 Register InsertedDst =
1557 NeedInsertSubvec ? MRI.createGenericVirtualRegister(Ty: InterLitTy) : Dst;
1558 if (RemIdx == 0) {
1559 Inserted = MIB.buildInstr(Opc: RISCV::G_VMV_V_V_VL, DstOps: {InsertedDst},
1560 SrcOps: {AlignedExtract, Insert, VL});
1561 } else {
1562 auto SlideupAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1563 // Construct the vector length corresponding to RemIdx + length(LitTy).
1564 VL = MIB.buildAdd(Dst: XLenTy, Src0: SlideupAmt, Src1: VL);
1565 // Use tail agnostic policy if we're inserting over InterLitTy's tail.
1566 ElementCount EndIndex =
1567 ElementCount::getScalable(MinVal: RemIdx) + LitTy.getElementCount();
1568 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
1569 if (STI.expandVScale(X: EndIndex) ==
1570 STI.expandVScale(X: InterLitTy.getElementCount()))
1571 Policy = RISCVVType::TAIL_AGNOSTIC;
1572
1573 Inserted =
1574 MIB.buildInstr(Opc: RISCV::G_VSLIDEUP_VL, DstOps: {InsertedDst},
1575 SrcOps: {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});
1576 }
1577
1578 // If required, insert this subvector back into the correct vector register.
1579 // This should resolve to an INSERT_SUBREG instruction.
1580 if (NeedInsertSubvec)
1581 MIB.buildInsertSubvector(Res: Dst, Src0: BigVec, Src1: Inserted, Index: AlignedIdx);
1582
1583 MI.eraseFromParent();
1584 return true;
1585}
1586
1587bool RISCVLegalizerInfo::legalizeBitreverse(MachineInstr &MI,
1588 MachineIRBuilder &MIB) const {
1589 assert(MI.getOpcode() == TargetOpcode::G_BITREVERSE && "Unexpected opcode");
1590
1591 if (!STI.hasStdExtZbkb())
1592 return false;
1593
1594 MachineRegisterInfo &MRI = *MIB.getMRI();
1595
1596 Register Dst = MI.getOperand(i: 0).getReg();
1597 Register Src = MI.getOperand(i: 1).getReg();
1598
1599 if (!MRI.getType(Reg: Dst).isScalar(Size: 8))
1600 return false;
1601
1602 auto WideSrc = MIB.buildAnyExt(Res: sXLen, Op: Src);
1603 auto Brev = MIB.buildInstr(Opc: RISCV::G_BREV8, DstOps: {sXLen}, SrcOps: {WideSrc.getReg(Idx: 0)});
1604 MIB.buildTrunc(Res: Dst, Op: Brev.getReg(Idx: 0));
1605
1606 MI.eraseFromParent();
1607 return true;
1608}
1609
1610static unsigned getRISCVWOpcode(unsigned Opcode) {
1611 switch (Opcode) {
1612 default:
1613 llvm_unreachable("Unexpected opcode");
1614 case TargetOpcode::G_ASHR:
1615 return RISCV::G_SRAW;
1616 case TargetOpcode::G_LSHR:
1617 return RISCV::G_SRLW;
1618 case TargetOpcode::G_SHL:
1619 return RISCV::G_SLLW;
1620 case TargetOpcode::G_SDIV:
1621 return RISCV::G_DIVW;
1622 case TargetOpcode::G_UDIV:
1623 return RISCV::G_DIVUW;
1624 case TargetOpcode::G_UREM:
1625 return RISCV::G_REMUW;
1626 case TargetOpcode::G_ROTL:
1627 return RISCV::G_ROLW;
1628 case TargetOpcode::G_ROTR:
1629 return RISCV::G_RORW;
1630 case TargetOpcode::G_CTLZ:
1631 return RISCV::G_CLZW;
1632 case TargetOpcode::G_CTTZ:
1633 return RISCV::G_CTZW;
1634 case TargetOpcode::G_CTLS:
1635 return RISCV::G_CLSW;
1636 case TargetOpcode::G_FPTOSI:
1637 return RISCV::G_FCVT_W_RV64;
1638 case TargetOpcode::G_FPTOUI:
1639 return RISCV::G_FCVT_WU_RV64;
1640 }
1641}
1642
1643bool RISCVLegalizerInfo::legalizeCustom(
1644 LegalizerHelper &Helper, MachineInstr &MI,
1645 LostDebugLocObserver &LocObserver) const {
1646 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1647 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1648 MachineFunction &MF = *MI.getParent()->getParent();
1649 switch (MI.getOpcode()) {
1650 default:
1651 // No idea what to do.
1652 return false;
1653 case TargetOpcode::G_ABS:
1654 return Helper.lowerAbsToMaxNeg(MI);
1655 case TargetOpcode::G_CLMULH:
1656 case TargetOpcode::G_CLMULR: {
1657 assert(STI.is64Bit() &&
1658 MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32) &&
1659 "Unexpected custom legalization");
1660 // Shift both inputs by 32 so the full product has 64 trailing zeros.
1661 // Perform CLMULH or CLMULR on the shifted inputs, then extract the upper
1662 // 32 bits of the result.
1663 auto Shift = MIRBuilder.buildConstant(Res: sXLen, Val: 32);
1664 auto LHS = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 1));
1665 auto RHS = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 2));
1666 auto ShiftedLHS = MIRBuilder.buildShl(Dst: sXLen, Src0: LHS, Src1: Shift);
1667 auto ShiftedRHS = MIRBuilder.buildShl(Dst: sXLen, Src0: RHS, Src1: Shift);
1668 auto Product = MIRBuilder.buildInstr(Opc: MI.getOpcode(), DstOps: {sXLen},
1669 SrcOps: {ShiftedLHS, ShiftedRHS});
1670 auto High = MIRBuilder.buildLShr(Dst: sXLen, Src0: Product, Src1: Shift);
1671 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: High);
1672 MI.eraseFromParent();
1673 return true;
1674 }
1675 case TargetOpcode::G_FCONSTANT: {
1676 const APFloat &FVal = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1677
1678 // Convert G_FCONSTANT to G_CONSTANT.
1679 Register DstReg = MI.getOperand(i: 0).getReg();
1680 MIRBuilder.buildConstant(Res: DstReg, Val: FVal.bitcastToAPInt());
1681
1682 MI.eraseFromParent();
1683 return true;
1684 }
1685 case TargetOpcode::G_CONSTANT: {
1686 const Function &F = MF.getFunction();
1687 // TODO: if PSI and BFI are present, add " ||
1688 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".
1689 bool ShouldOptForSize = F.hasOptSize();
1690 const ConstantInt *ConstVal = MI.getOperand(i: 1).getCImm();
1691 if (!shouldBeInConstantPool(APImm: ConstVal->getValue(), ShouldOptForSize))
1692 return true;
1693 return Helper.lowerConstant(MI);
1694 }
1695 case TargetOpcode::G_SUB:
1696 case TargetOpcode::G_ADD: {
1697 Helper.Observer.changingInstr(MI);
1698 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1699 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1700
1701 Register DstALU = MRI.createGenericVirtualRegister(Ty: sXLen);
1702
1703 MachineOperand &MO = MI.getOperand(i: 0);
1704 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
1705 auto DstSext = MIRBuilder.buildSExtInReg(Res: sXLen, Op: DstALU, ImmOp: 32);
1706
1707 MIRBuilder.buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: {MO}, SrcOps: {DstSext});
1708 MO.setReg(DstALU);
1709
1710 Helper.Observer.changedInstr(MI);
1711 return true;
1712 }
1713 case TargetOpcode::G_ASHR:
1714 case TargetOpcode::G_LSHR:
1715 case TargetOpcode::G_SHL: {
1716 if (getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI)) {
1717 // We don't need a custom node for shift by constant. Just widen the
1718 // source and the shift amount.
1719 unsigned ExtOpc = TargetOpcode::G_ANYEXT;
1720 if (MI.getOpcode() == TargetOpcode::G_ASHR)
1721 ExtOpc = TargetOpcode::G_SEXT;
1722 else if (MI.getOpcode() == TargetOpcode::G_LSHR)
1723 ExtOpc = TargetOpcode::G_ZEXT;
1724
1725 Helper.Observer.changingInstr(MI);
1726 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: ExtOpc);
1727 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
1728 Helper.widenScalarDst(MI, WideTy: sXLen);
1729 Helper.Observer.changedInstr(MI);
1730 return true;
1731 }
1732
1733 Helper.Observer.changingInstr(MI);
1734 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1735 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1736 Helper.widenScalarDst(MI, WideTy: sXLen);
1737 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1738 Helper.Observer.changedInstr(MI);
1739 return true;
1740 }
1741 case TargetOpcode::G_SDIV:
1742 case TargetOpcode::G_UDIV:
1743 case TargetOpcode::G_UREM:
1744 case TargetOpcode::G_ROTL:
1745 case TargetOpcode::G_ROTR: {
1746 Helper.Observer.changingInstr(MI);
1747 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1748 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1749 Helper.widenScalarDst(MI, WideTy: sXLen);
1750 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1751 Helper.Observer.changedInstr(MI);
1752 return true;
1753 }
1754 case TargetOpcode::G_CTLZ:
1755 case TargetOpcode::G_CTTZ:
1756 case TargetOpcode::G_CTLS: {
1757 Helper.Observer.changingInstr(MI);
1758 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1759 Helper.widenScalarDst(MI, WideTy: sXLen);
1760 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1761 Helper.Observer.changedInstr(MI);
1762 return true;
1763 }
1764 case TargetOpcode::G_FPTOSI:
1765 case TargetOpcode::G_FPTOUI: {
1766 Helper.Observer.changingInstr(MI);
1767 Helper.widenScalarDst(MI, WideTy: sXLen);
1768 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1769 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RTZ));
1770 Helper.Observer.changedInstr(MI);
1771 return true;
1772 }
1773 case TargetOpcode::G_LROUND: {
1774 // The (i32 any_lround) Pat is IsRV32-only; on RV64 lower to
1775 // riscv_fcvt_w_rv64 with FRM_RMM.
1776 Helper.Observer.changingInstr(MI);
1777 Helper.widenScalarDst(MI, WideTy: sXLen);
1778 MI.setDesc(MIRBuilder.getTII().get(Opcode: RISCV::G_FCVT_W_RV64));
1779 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RMM));
1780 Helper.Observer.changedInstr(MI);
1781 return true;
1782 }
1783 case TargetOpcode::G_READCYCLECOUNTER:
1784 case TargetOpcode::G_READSTEADYCOUNTER:
1785 return legalizeReadCounter(MI, MIRBuilder, Observer&: Helper.Observer);
1786 case TargetOpcode::G_IS_FPCLASS: {
1787 Register GISFPCLASS = MI.getOperand(i: 0).getReg();
1788 Register Src = MI.getOperand(i: 1).getReg();
1789 const MachineOperand &ImmOp = MI.getOperand(i: 2);
1790 MachineIRBuilder MIB(MI);
1791
1792 // Turn LLVM IR's floating point classes to that in RISC-V,
1793 // by simply rotating the 10-bit immediate right by two bits.
1794 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));
1795 auto FClassMask = MIB.buildConstant(Res: sXLen, Val: GFpClassImm.rotr(rotateAmt: 2).zext(width: XLen));
1796 auto ConstZero = MIB.buildConstant(Res: sXLen, Val: 0);
1797
1798 auto GFClass = MIB.buildInstr(Opc: RISCV::G_FCLASS, DstOps: {sXLen}, SrcOps: {Src});
1799 auto And = MIB.buildAnd(Dst: sXLen, Src0: GFClass, Src1: FClassMask);
1800 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: GISFPCLASS, Op0: And, Op1: ConstZero);
1801
1802 MI.eraseFromParent();
1803 return true;
1804 }
1805 case TargetOpcode::G_BRJT:
1806 return legalizeBRJT(MI, MIRBuilder);
1807 case TargetOpcode::G_VASTART:
1808 return legalizeVAStart(MI, MIRBuilder);
1809 case TargetOpcode::G_VSCALE:
1810 return legalizeVScale(MI, MIB&: MIRBuilder);
1811 case TargetOpcode::G_ZEXT:
1812 case TargetOpcode::G_SEXT:
1813 case TargetOpcode::G_ANYEXT:
1814 return legalizeExt(MI, MIB&: MIRBuilder);
1815 case TargetOpcode::G_SPLAT_VECTOR:
1816 return legalizeSplatVector(MI, MIB&: MIRBuilder);
1817 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1818 return legalizeExtractSubvector(MI, MIB&: MIRBuilder);
1819 case TargetOpcode::G_INSERT_SUBVECTOR:
1820 return legalizeInsertSubvector(MI, Helper, MIB&: MIRBuilder);
1821 case TargetOpcode::G_BITREVERSE:
1822 return legalizeBitreverse(MI, MIB&: MIRBuilder);
1823 case TargetOpcode::G_LOAD:
1824 case TargetOpcode::G_STORE:
1825 return legalizeLoadStore(MI, Helper, MIB&: MIRBuilder);
1826 }
1827
1828 llvm_unreachable("expected switch to return");
1829}
1830