1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for RISC-V.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "RISCVLegalizerInfo.h"
14#include "MCTargetDesc/RISCVMatInt.h"
15#include "RISCVMachineFunctionInfo.h"
16#include "RISCVSubtarget.h"
17#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/MachineConstantPool.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineMemOperand.h"
25#include "llvm/CodeGen/MachineOperand.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/TargetOpcodes.h"
28#include "llvm/CodeGen/ValueTypes.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsRISCV.h"
32#include "llvm/IR/Type.h"
33
34using namespace llvm;
35using namespace LegalityPredicates;
36using namespace LegalizeMutations;
37using namespace MIPatternMatch;
38
39static LegalityPredicate
40typeIsLegalIntOrFPVec(unsigned TypeIdx,
41 std::initializer_list<LLT> IntOrFPVecTys,
42 const RISCVSubtarget &ST) {
43 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
44 return ST.hasVInstructions() &&
45 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
46 ST.hasVInstructionsI64()) &&
47 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
48 ST.getELen() == 64);
49 };
50
51 return all(P0: typeInSet(TypeIdx, TypesInit: IntOrFPVecTys), P1: P);
52}
53
54static LegalityPredicate
55typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
56 const RISCVSubtarget &ST) {
57 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
58 return ST.hasVInstructions() &&
59 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
60 ST.getELen() == 64);
61 };
62 return all(P0: typeInSet(TypeIdx, TypesInit: BoolVecTys), P1: P);
63}
64
65static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
66 std::initializer_list<LLT> PtrVecTys,
67 const RISCVSubtarget &ST) {
68 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
69 return ST.hasVInstructions() &&
70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
71 ST.getELen() == 64) &&
72 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
73 Query.Types[TypeIdx].getScalarSizeInBits() == 32);
74 };
75 return all(P0: typeInSet(TypeIdx, TypesInit: PtrVecTys), P1: P);
76}
77
78RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
79 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(SizeInBits: XLen)) {
80 const LLT sDoubleXLen = LLT::scalar(SizeInBits: 2 * XLen);
81 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: XLen);
82 const LLT s1 = LLT::scalar(SizeInBits: 1);
83 const LLT s8 = LLT::scalar(SizeInBits: 8);
84 const LLT s16 = LLT::scalar(SizeInBits: 16);
85 const LLT s32 = LLT::scalar(SizeInBits: 32);
86 const LLT s64 = LLT::scalar(SizeInBits: 64);
87 const LLT s128 = LLT::scalar(SizeInBits: 128);
88
89 const LLT nxv1s1 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s1);
90 const LLT nxv2s1 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s1);
91 const LLT nxv4s1 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s1);
92 const LLT nxv8s1 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s1);
93 const LLT nxv16s1 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s1);
94 const LLT nxv32s1 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s1);
95 const LLT nxv64s1 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s1);
96
97 const LLT nxv1s8 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s8);
98 const LLT nxv2s8 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s8);
99 const LLT nxv4s8 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s8);
100 const LLT nxv8s8 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s8);
101 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
102 const LLT nxv32s8 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s8);
103 const LLT nxv64s8 = LLT::scalable_vector(MinNumElements: 64, ScalarTy: s8);
104
105 const LLT nxv1s16 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s16);
106 const LLT nxv2s16 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s16);
107 const LLT nxv4s16 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s16);
108 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
109 const LLT nxv16s16 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s16);
110 const LLT nxv32s16 = LLT::scalable_vector(MinNumElements: 32, ScalarTy: s16);
111
112 const LLT nxv1s32 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s32);
113 const LLT nxv2s32 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s32);
114 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
115 const LLT nxv8s32 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s32);
116 const LLT nxv16s32 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s32);
117
118 const LLT nxv1s64 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: s64);
119 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
120 const LLT nxv4s64 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s64);
121 const LLT nxv8s64 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s64);
122
123 const LLT nxv1p0 = LLT::scalable_vector(MinNumElements: 1, ScalarTy: p0);
124 const LLT nxv2p0 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: p0);
125 const LLT nxv4p0 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: p0);
126 const LLT nxv8p0 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: p0);
127 const LLT nxv16p0 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: p0);
128
129 using namespace TargetOpcode;
130
131 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
132
133 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,
134 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
135 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
136 nxv1s64, nxv2s64, nxv4s64, nxv8s64};
137
138 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
139
140 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
141 .legalFor(Types: {sXLen})
142 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
143 .customFor(Pred: ST.is64Bit(), Types: {s32})
144 .widenScalarToNextPow2(TypeIdx: 0)
145 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
146
147 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
148 .legalFor(Types: {sXLen})
149 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
150 .widenScalarToNextPow2(TypeIdx: 0)
151 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
152
153 getActionDefinitionsBuilder(
154 Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO, G_READ_REGISTER, G_WRITE_REGISTER})
155 .lower();
156
157 getActionDefinitionsBuilder(Opcodes: {G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})
158 .minScalar(TypeIdx: 0, Ty: sXLen)
159 .lower();
160
161 // TODO: Use Vector Single-Width Saturating Instructions for vector types.
162 getActionDefinitionsBuilder(
163 Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})
164 .lower();
165
166 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
167 .legalFor(Types: {{sXLen, sXLen}})
168 .customFor(Pred: ST.is64Bit(), Types: {{s32, s32}})
169 .widenScalarToNextPow2(TypeIdx: 0)
170 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
171 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
172
173 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
174 .legalFor(Types: {{s32, s16}})
175 .legalFor(Pred: ST.is64Bit(), Types: {{s64, s16}, {s64, s32}})
176 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
177 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
178 .customIf(Predicate: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))
179 .maxScalar(TypeIdx: 0, Ty: sXLen);
180
181 getActionDefinitionsBuilder(Opcode: G_TRUNC).alwaysLegal();
182
183 {
184 LegalityPredicate ValidSextInRegWidth = all(P0: sizeIs(TypeIdx: 0, Size: 64), P1: immIs(ImmIdx: 0, Imm: 32));
185
186 if (STI.hasStdExtZbb())
187 ValidSextInRegWidth =
188 LegalityPredicates::any(P0: ValidSextInRegWidth, P1: immInSet(ImmIdx: 0, ImmsInit: {8, 16}));
189
190 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
191 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: sXLen), P1: ValidSextInRegWidth))
192 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
193 .lower();
194 }
195
196 // Merge/Unmerge
197 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
198 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Opcode: Op);
199 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
200 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
201 if (XLen == 32 && ST.hasStdExtD()) {
202 MergeUnmergeActions.legalIf(
203 Predicate: all(P0: typeIs(TypeIdx: BigTyIdx, TypesInit: s64), P1: typeIs(TypeIdx: LitTyIdx, TypesInit: s32)));
204 }
205 MergeUnmergeActions.widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: XLen)
206 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: XLen)
207 .clampScalar(TypeIdx: LitTyIdx, MinTy: sXLen, MaxTy: sXLen)
208 .clampScalar(TypeIdx: BigTyIdx, MinTy: sXLen, MaxTy: sXLen);
209 }
210
211 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR}).lower();
212
213 getActionDefinitionsBuilder(Opcodes: {G_ROTR, G_ROTL})
214 .legalFor(Pred: ST.hasStdExtZbb() || ST.hasStdExtZbkb(), Types: {{sXLen, sXLen}})
215 .customFor(Pred: ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),
216 Types: {{s32, s32}})
217 .lower();
218
219 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
220 .customFor(Pred: ST.hasStdExtZbkb(), Types: {s8})
221 .maxScalar(TypeIdx: 0, Ty: sXLen)
222 .lower();
223
224 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
225 Predicate: all(P0: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
226 P1: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST)),
227 P1: LegalityPredicates::any(P0: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST),
228 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST))));
229
230 auto &BSWAPActions = getActionDefinitionsBuilder(Opcode: G_BSWAP);
231 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
232 BSWAPActions.legalFor(Types: {sXLen}).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
233 else
234 BSWAPActions.maxScalar(TypeIdx: 0, Ty: sXLen).lower();
235
236 getActionDefinitionsBuilder(Opcode: G_CLMUL)
237 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
238 .unsupported();
239
240 getActionDefinitionsBuilder(Opcode: G_CLMULH)
241 .legalFor(Pred: ST.hasStdExtZbkc(), Types: {sXLen})
242 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZbkc(), Types: {s32})
243 .unsupported();
244
245 auto &CountZerosActions = getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTTZ});
246 auto &CountZerosPoisonActions =
247 getActionDefinitionsBuilder(Opcodes: {G_CTLZ_ZERO_POISON, G_CTTZ_ZERO_POISON});
248 if (ST.hasStdExtZbb()) {
249 CountZerosActions.legalFor(Types: {{sXLen, sXLen}})
250 .customFor(Types: {{s32, s32}})
251 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
252 .widenScalarToNextPow2(TypeIdx: 0)
253 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
254 } else {
255 CountZerosActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
256 CountZerosPoisonActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
257 }
258 CountZerosPoisonActions.lower();
259
260 auto &CountSignActions = getActionDefinitionsBuilder(Opcode: G_CTLS);
261 if (ST.hasStdExtP()) {
262 CountSignActions.legalFor(Types: {{sXLen, sXLen}})
263 .customFor(Types: {{s32, s32}})
264 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen)
265 .widenScalarToNextPow2(TypeIdx: 0)
266 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
267 } else {
268 CountSignActions.maxScalar(TypeIdx: 0, Ty: sXLen).scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0).lower();
269 }
270
271 auto &CTPOPActions = getActionDefinitionsBuilder(Opcode: G_CTPOP);
272 if (ST.hasStdExtZbb()) {
273 CTPOPActions.legalFor(Types: {{sXLen, sXLen}})
274 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
275 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0);
276 } else {
277 CTPOPActions.widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8)
278 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sXLen)
279 .scalarSameSizeAs(TypeIdx: 1, SameSizeIdx: 0)
280 .lower();
281 }
282
283 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
284 .legalFor(Types: {p0})
285 .legalFor(Pred: !ST.is64Bit(), Types: {s32})
286 .customFor(Pred: ST.is64Bit(), Types: {s64})
287 .widenScalarToNextPow2(TypeIdx: 0)
288 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
289
290 // TODO: transform illegal vector types into legal vector type
291 getActionDefinitionsBuilder(Opcode: G_FREEZE)
292 .legalFor(Types: {s16, s32, p0})
293 .legalFor(Pred: ST.is64Bit(), Types: {s64})
294 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
295 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
296 .widenScalarToNextPow2(TypeIdx: 0)
297 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen);
298
299 // TODO: transform illegal vector types into legal vector type
300 // TODO: Merge with G_FREEZE?
301 getActionDefinitionsBuilder(
302 Opcodes: {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})
303 .legalFor(Types: {s32, sXLen, p0})
304 .legalIf(Predicate: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST))
305 .legalIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST))
306 .widenScalarToNextPow2(TypeIdx: 0)
307 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
308
309 getActionDefinitionsBuilder(Opcode: G_ICMP)
310 .legalFor(Types: {{sXLen, sXLen}, {sXLen, p0}})
311 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
312 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)))
313 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, MinSize: 8)
314 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen)
315 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
316
317 getActionDefinitionsBuilder(Opcode: G_SELECT)
318 .legalFor(Types: {{s32, sXLen}, {p0, sXLen}})
319 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
320 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
321 .legalFor(Pred: XLen == 64 || ST.hasStdExtD(), Types: {{s64, sXLen}})
322 .widenScalarToNextPow2(TypeIdx: 0)
323 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)
324 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
325
326 auto &LoadActions = getActionDefinitionsBuilder(Opcode: G_LOAD);
327 auto &StoreActions = getActionDefinitionsBuilder(Opcode: G_STORE);
328 auto &ExtLoadActions = getActionDefinitionsBuilder(Opcodes: {G_SEXTLOAD, G_ZEXTLOAD});
329
330 // Return the alignment needed for scalar memory ops. If unaligned scalar mem
331 // is supported, we only require byte alignment. Otherwise, we need the memory
332 // op to be natively aligned.
333 auto getScalarMemAlign = [&ST](unsigned Size) {
334 return ST.enableUnalignedScalarMem() ? 8 : Size;
335 };
336
337 LoadActions.legalForTypesWithMemDesc(
338 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
339 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
340 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
341 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
342 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
343 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
344 StoreActions.legalForTypesWithMemDesc(
345 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
346 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
347 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
348 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
349 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
350 {.Type0: p0, .Type1: p0, .MemTy: sXLen, .Align: getScalarMemAlign(XLen)}});
351 ExtLoadActions.legalForTypesWithMemDesc(
352 TypesAndMemDesc: {{.Type0: sXLen, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
353 {.Type0: sXLen, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)}});
354 if (XLen == 64) {
355 LoadActions.legalForTypesWithMemDesc(
356 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
357 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
358 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
359 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
360 StoreActions.legalForTypesWithMemDesc(
361 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: getScalarMemAlign(8)},
362 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: getScalarMemAlign(16)},
363 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)},
364 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
365 ExtLoadActions.legalForTypesWithMemDesc(
366 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s32, .Align: getScalarMemAlign(32)}});
367 } else if (ST.hasStdExtD()) {
368 LoadActions.legalForTypesWithMemDesc(
369 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
370 StoreActions.legalForTypesWithMemDesc(
371 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: getScalarMemAlign(64)}});
372 }
373
374 // Vector loads/stores.
375 if (ST.hasVInstructions()) {
376 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
377 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
378 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
379 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
380 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
381 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
382 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
383 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
384 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
385 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
386 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
387 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
388 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
389 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
390 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
391 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv2s8, .Type1: p0, .MemTy: nxv2s8, .Align: 8},
392 {.Type0: nxv4s8, .Type1: p0, .MemTy: nxv4s8, .Align: 8},
393 {.Type0: nxv8s8, .Type1: p0, .MemTy: nxv8s8, .Align: 8},
394 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
395 {.Type0: nxv32s8, .Type1: p0, .MemTy: nxv32s8, .Align: 8},
396 {.Type0: nxv64s8, .Type1: p0, .MemTy: nxv64s8, .Align: 8},
397 {.Type0: nxv2s16, .Type1: p0, .MemTy: nxv2s16, .Align: 16},
398 {.Type0: nxv4s16, .Type1: p0, .MemTy: nxv4s16, .Align: 16},
399 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 16},
400 {.Type0: nxv16s16, .Type1: p0, .MemTy: nxv16s16, .Align: 16},
401 {.Type0: nxv32s16, .Type1: p0, .MemTy: nxv32s16, .Align: 16},
402 {.Type0: nxv2s32, .Type1: p0, .MemTy: nxv2s32, .Align: 32},
403 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 32},
404 {.Type0: nxv8s32, .Type1: p0, .MemTy: nxv8s32, .Align: 32},
405 {.Type0: nxv16s32, .Type1: p0, .MemTy: nxv16s32, .Align: 32}});
406
407 if (ST.getELen() == 64) {
408 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
409 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
410 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
411 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s8, .Type1: p0, .MemTy: nxv1s8, .Align: 8},
412 {.Type0: nxv1s16, .Type1: p0, .MemTy: nxv1s16, .Align: 16},
413 {.Type0: nxv1s32, .Type1: p0, .MemTy: nxv1s32, .Align: 32}});
414 }
415
416 if (ST.hasVInstructionsI64()) {
417 LoadActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
418 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
419 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
420 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
421 StoreActions.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: nxv1s64, .Type1: p0, .MemTy: nxv1s64, .Align: 64},
422 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 64},
423 {.Type0: nxv4s64, .Type1: p0, .MemTy: nxv4s64, .Align: 64},
424 {.Type0: nxv8s64, .Type1: p0, .MemTy: nxv8s64, .Align: 64}});
425 }
426
427 // we will take the custom lowering logic if we have scalable vector types
428 // with non-standard alignments
429 LoadActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
430 StoreActions.customIf(Predicate: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST));
431
432 // Pointers require that XLen sized elements are legal.
433 if (XLen <= ST.getELen()) {
434 LoadActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
435 StoreActions.customIf(Predicate: typeIsLegalPtrVec(TypeIdx: 0, PtrVecTys, ST));
436 }
437 }
438
439 LoadActions.widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
440 .lowerIfMemSizeNotByteSizePow2()
441 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
442 .lower();
443 StoreActions
444 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: sXLen)
445 .lowerIfMemSizeNotByteSizePow2()
446 .lower();
447
448 ExtLoadActions.widenScalarToNextPow2(TypeIdx: 0).clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen).lower();
449
450 getActionDefinitionsBuilder(Opcodes: {G_PTR_ADD, G_PTRMASK}).legalFor(Types: {{p0, sXLen}});
451
452 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
453 .legalFor(Types: {{sXLen, p0}})
454 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
455
456 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
457 .legalFor(Types: {{p0, sXLen}})
458 .clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
459
460 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
461
462 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {sXLen}).minScalar(TypeIdx: 0, Ty: sXLen);
463
464 getActionDefinitionsBuilder(Opcode: G_BRJT).customFor(Types: {{p0, sXLen}});
465
466 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
467
468 getActionDefinitionsBuilder(Opcode: G_PHI)
469 .legalFor(Types: {p0, s32, sXLen})
470 .widenScalarToNextPow2(TypeIdx: 0)
471 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sXLen);
472
473 getActionDefinitionsBuilder(Opcodes: {G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
474 .legalFor(Types: {p0});
475
476 if (ST.hasStdExtZmmul()) {
477 getActionDefinitionsBuilder(Opcode: G_MUL)
478 .legalFor(Types: {sXLen})
479 .widenScalarToNextPow2(TypeIdx: 0)
480 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
481
482 // clang-format off
483 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
484 .legalFor(Types: {sXLen})
485 .lower();
486 // clang-format on
487
488 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO}).minScalar(TypeIdx: 0, Ty: sXLen).lower();
489 } else {
490 getActionDefinitionsBuilder(Opcode: G_MUL)
491 .libcallFor(Types: {sXLen, sDoubleXLen})
492 .widenScalarToNextPow2(TypeIdx: 0)
493 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen);
494
495 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).lowerFor(Types: {sXLen});
496
497 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
498 .minScalar(TypeIdx: 0, Ty: sXLen)
499 // Widen sXLen to sDoubleXLen so we can use a single libcall to get
500 // the low bits for the mul result and high bits to do the overflow
501 // check.
502 .widenScalarIf(Predicate: typeIs(TypeIdx: 0, TypesInit: sXLen),
503 Mutation: LegalizeMutations::changeTo(TypeIdx: 0, Ty: sDoubleXLen))
504 .lower();
505 }
506
507 if (ST.hasStdExtM()) {
508 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV, G_UREM})
509 .legalFor(Types: {sXLen})
510 .customFor(Types: {s32})
511 .libcallFor(Types: {sDoubleXLen})
512 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sDoubleXLen)
513 .widenScalarToNextPow2(TypeIdx: 0);
514 getActionDefinitionsBuilder(Opcode: G_SREM)
515 .legalFor(Types: {sXLen})
516 .libcallFor(Types: {sDoubleXLen})
517 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
518 .widenScalarToNextPow2(TypeIdx: 0);
519 } else {
520 getActionDefinitionsBuilder(Opcodes: {G_UDIV, G_SDIV, G_UREM, G_SREM})
521 .libcallFor(Types: {sXLen, sDoubleXLen})
522 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sDoubleXLen)
523 .widenScalarToNextPow2(TypeIdx: 0);
524 }
525
526 // TODO: Use libcall for sDoubleXLen.
527 getActionDefinitionsBuilder(Opcodes: {G_SDIVREM, G_UDIVREM}).lower();
528
529 getActionDefinitionsBuilder(Opcode: G_ABS)
530 .customFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
531 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
532 .lower();
533
534 getActionDefinitionsBuilder(Opcodes: {G_ABDS, G_ABDU})
535 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
536 .lower();
537
538 getActionDefinitionsBuilder(Opcodes: {G_UMAX, G_UMIN, G_SMAX, G_SMIN})
539 .legalFor(Pred: ST.hasStdExtZbb(), Types: {sXLen})
540 .minScalar(Pred: ST.hasStdExtZbb(), TypeIdx: 0, Ty: sXLen)
541 .lower();
542
543 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
544
545 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
546
547 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
548
549 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
550
551 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
552 .lower();
553
554 // On RV64 the 64-bit counter CSRs (cycle/time) are read directly. On RV32
555 // they are custom-legally lowered to a re-read-the-high-half loop (see
556 // legalizeReadCounter).
557 getActionDefinitionsBuilder(Opcodes: {G_READCYCLECOUNTER, G_READSTEADYCOUNTER})
558 .legalFor(Pred: ST.is64Bit(), Types: {s64})
559 .customFor(Pred: !ST.is64Bit(), Types: {s64});
560
561 // FP Operations
562
563 // FIXME: Support s128 for rv32 when libcall handling is able to use sret.
564 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,
565 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,
566 G_FMINIMUMNUM})
567 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
568 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
569 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
570 .libcallFor(Types: {s32, s64})
571 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
572
573 getActionDefinitionsBuilder(Opcodes: {G_FNEG, G_FABS})
574 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
575 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
576 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
577 .lowerFor(Types: {s32, s64, s128});
578
579 getActionDefinitionsBuilder(Opcode: G_FREM)
580 .libcallFor(Types: {s32, s64})
581 .libcallFor(Pred: ST.is64Bit(), Types: {s128})
582 .minScalar(TypeIdx: 0, Ty: s32)
583 .scalarize(TypeIdx: 0);
584
585 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
586 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, s32}})
587 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s64}, {s32, s64}, {s64, s32}})
588 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s16}, {s16, s32}, {s32, s16}})
589 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}, {s64, s16}})
590 .lower();
591
592 // FIXME: Use Zfhmin.
593 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
594 .legalFor(Pred: ST.hasStdExtD(), Types: {{s32, s64}})
595 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, s32}})
596 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s16, s64}})
597 .libcallFor(Types: {{s32, s64}})
598 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}});
599 getActionDefinitionsBuilder(Opcode: G_FPEXT)
600 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, s32}})
601 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s32, s16}})
602 .legalFor(Pred: ST.hasStdExtZfh() && ST.hasStdExtD(), Types: {{s64, s16}})
603 .libcallFor(Types: {{s64, s32}})
604 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}});
605
606 getActionDefinitionsBuilder(Opcode: G_FCMP)
607 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
608 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
609 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
610 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
611 .libcallFor(Types: {{sXLen, s32}, {sXLen, s64}})
612 .libcallFor(Pred: ST.is64Bit(), Types: {{sXLen, s128}});
613
614 // TODO: Support vector version of G_IS_FPCLASS.
615 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
616 .customFor(Pred: ST.hasStdExtF(), Types: {{s1, s32}})
617 .customFor(Pred: ST.hasStdExtD(), Types: {{s1, s64}})
618 .customFor(Pred: ST.hasStdExtZfh(), Types: {{s1, s16}})
619 .lower();
620
621 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
622 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
623 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
624 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16})
625 .customFor(Pred: !ST.is64Bit(), Types: {s32})
626 .customFor(Pred: ST.is64Bit(), Types: {s32, s64})
627 .lowerFor(Types: {s64, s128});
628
629 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
630 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
631 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
632 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
633 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
634 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
635 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
636 .widenScalarToNextPow2(TypeIdx: 0)
637 .minScalar(TypeIdx: 0, Ty: s32)
638 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
639 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}}) // FIXME RV32.
640 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}, {s128, s128}});
641
642 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
643 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
644 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
645 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
646 .customFor(Pred: ST.is64Bit() && ST.hasStdExtF(), Types: {{s32, s32}})
647 .customFor(Pred: ST.is64Bit() && ST.hasStdExtD(), Types: {{s32, s64}})
648 .customFor(Pred: ST.is64Bit() && ST.hasStdExtZfh(), Types: {{s32, s16}})
649 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16), Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: s32))
650 .libcallFor(Types: {{s32, s32},
651 {s64, s32},
652 {s32, s64},
653 {s64, s64},
654 {s32, s128},
655 {s64, s128}});
656
657 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
658 .legalFor(Pred: ST.hasStdExtF(), Types: {{sXLen, s32}})
659 .legalFor(Pred: ST.hasStdExtD(), Types: {{sXLen, s64}})
660 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{sXLen, s16}})
661 .minScalar(TypeIdx: 0, Ty: sXLen)
662 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16), Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: s32))
663 .libcallFor(Types: {{s32, s32},
664 {s64, s32},
665 {s32, s64},
666 {s64, s64},
667 {s32, s128},
668 {s64, s128}});
669
670 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
671 .legalFor(Pred: ST.hasStdExtF(), Types: {{s32, sXLen}})
672 .legalFor(Pred: ST.hasStdExtD(), Types: {{s64, sXLen}})
673 .legalFor(Pred: ST.hasStdExtZfh(), Types: {{s16, sXLen}})
674 .widenScalarToNextPow2(TypeIdx: 1)
675 // Promote to XLen if the operation is legal.
676 .widenScalarIf(
677 Predicate: [=, &ST](const LegalityQuery &Query) {
678 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
679 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
680 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
681 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
682 (ST.hasStdExtZfh() &&
683 Query.Types[0].getSizeInBits() == 16));
684 },
685 Mutation: LegalizeMutations::changeTo(TypeIdx: 1, Ty: sXLen))
686 // Otherwise only promote to s32 since we have si libcalls.
687 .minScalar(TypeIdx: 1, Ty: s32)
688 .libcallFor(Types: {{s32, s32}, {s64, s32}, {s32, s64}, {s64, s64}})
689 .libcallFor(Pred: ST.is64Bit(), Types: {{s128, s32}, {s128, s64}}) // FIXME RV32.
690 .libcallFor(Pred: ST.is64Bit(), Types: {{s32, s128}, {s64, s128}, {s128, s128}});
691
692 // FIXME: We can do custom inline expansion like SelectionDAG.
693 getActionDefinitionsBuilder(Opcodes: {G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
694 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
695 G_INTRINSIC_ROUNDEVEN})
696 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
697 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
698 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16})
699 .libcallFor(Types: {s32, s64})
700 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
701
702 getActionDefinitionsBuilder(Opcodes: {G_FMAXIMUM, G_FMINIMUM})
703 .legalFor(Pred: ST.hasStdExtZfa(), Types: {s32})
704 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtD(), Types: {s64})
705 .legalFor(Pred: ST.hasStdExtZfa() && ST.hasStdExtZfh(), Types: {s16});
706
707 getActionDefinitionsBuilder(Opcodes: {G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
708 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
709 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
710 G_FTANH, G_FMODF})
711 .libcallFor(Types: {s32, s64})
712 .libcallFor(Pred: ST.is64Bit(), Types: {s128});
713 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
714 .libcallFor(Types: {{s32, s32}, {s64, s32}})
715 .libcallFor(Pred: ST.is64Bit(), Types: {s128, s32});
716
717 getActionDefinitionsBuilder(Opcode: G_FCANONICALIZE)
718 .legalFor(Pred: ST.hasStdExtF(), Types: {s32})
719 .legalFor(Pred: ST.hasStdExtD(), Types: {s64})
720 .legalFor(Pred: ST.hasStdExtZfh(), Types: {s16});
721
722 getActionDefinitionsBuilder(Opcode: G_VASTART).customFor(Types: {p0});
723
724 // va_list must be a pointer, but most sized types are pretty easy to handle
725 // as the destination.
726 getActionDefinitionsBuilder(Opcode: G_VAARG)
727 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types
728 // other than sXLen.
729 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
730 .lowerForCartesianProduct(Types0: {sXLen, p0}, Types1: {p0});
731
732 getActionDefinitionsBuilder(Opcode: G_VSCALE)
733 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
734 .customFor(Types: {sXLen});
735
736 auto &SplatActions =
737 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
738 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
739 P1: typeIs(TypeIdx: 1, TypesInit: sXLen)))
740 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: typeIs(TypeIdx: 1, TypesInit: s1)));
741 // Handle case of s64 element vectors on RV32. If the subtarget does not have
742 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget
743 // does have f64, then we don't know whether the type is an f64 or an i64,
744 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,
745 // depending on how the instructions it consumes are legalized. They are not
746 // legalized yet since legalization is in reverse postorder, so we cannot
747 // make the decision at this moment.
748 if (XLen == 32) {
749 if (ST.hasVInstructionsF64() && ST.hasStdExtD())
750 SplatActions.legalIf(Predicate: all(
751 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
752 else if (ST.hasVInstructionsI64())
753 SplatActions.customIf(Predicate: all(
754 P0: typeInSet(TypeIdx: 0, TypesInit: {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), P1: typeIs(TypeIdx: 1, TypesInit: s64)));
755 }
756
757 SplatActions.clampScalar(TypeIdx: 1, MinTy: sXLen, MaxTy: sXLen);
758
759 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
760 LLT DstTy = Query.Types[0];
761 LLT SrcTy = Query.Types[1];
762 return DstTy.getElementType() == LLT::scalar(SizeInBits: 1) &&
763 DstTy.getElementCount().getKnownMinValue() >= 8 &&
764 SrcTy.getElementCount().getKnownMinValue() >= 8;
765 };
766 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
767 // We don't have the ability to slide mask vectors down indexed by their
768 // i1 elements; the smallest we can do is i8. Often we are able to bitcast
769 // to equivalent i8 vectors.
770 .bitcastIf(
771 Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
772 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST), args: ExtractSubvecBitcastPred),
773 Mutation: [=](const LegalityQuery &Query) {
774 LLT CastTy = LLT::vector(
775 EC: Query.Types[0].getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8);
776 return std::pair(0, CastTy);
777 })
778 .customIf(Predicate: LegalityPredicates::any(
779 P0: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
780 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)),
781 P1: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
782 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST))));
783
784 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR)
785 .customIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST),
786 P1: typeIsLegalBoolVec(TypeIdx: 1, BoolVecTys, ST)))
787 .customIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
788 P1: typeIsLegalIntOrFPVec(TypeIdx: 1, IntOrFPVecTys, ST)));
789
790 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
791 .lowerIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
792
793 getActionDefinitionsBuilder(Opcodes: {G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
794 G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
795 G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
796 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
797 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
798 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen);
799
800 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_SUB)
801 .libcallFor(Pred: !ST.hasStdExtA(), Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
802 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
803 .lower();
804
805 getActionDefinitionsBuilder(
806 Opcodes: {G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
807 .legalFor(Pred: ST.hasStdExtA(), Types: {{sXLen, p0}})
808 .clampScalar(TypeIdx: 0, MinTy: sXLen, MaxTy: sXLen)
809 .unsupported();
810
811 getActionDefinitionsBuilder(Opcode: G_PREFETCH).legalIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p0));
812
813 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
814 LLT VecTy = Query.Types[0];
815 LLT EltTy = Query.Types[1];
816 return VecTy.getElementType() == EltTy;
817 };
818
819 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
820 .legalIf(Predicate: all(P0: typeIsLegalIntOrFPVec(TypeIdx: 0, IntOrFPVecTys, ST),
821 P1: InsertVectorEltPred, args: typeIs(TypeIdx: 2, TypesInit: sXLen)))
822 .legalIf(Predicate: all(P0: typeIsLegalBoolVec(TypeIdx: 0, BoolVecTys, ST), P1: InsertVectorEltPred,
823 args: typeIs(TypeIdx: 2, TypesInit: sXLen)));
824
825 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
826 .alwaysLegal();
827
828 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
829
830 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
831
832 verify(MII: *ST.getInstrInfo());
833}
834
835bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
836 MachineInstr &MI) const {
837 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
838
839 if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
840 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {
841 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {
842 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
843 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
844
845 auto OldScalar = MI.getOperand(i: II->ScalarOperand + 2).getReg();
846 // Legalize integer vx form intrinsic.
847 if (MRI.getType(Reg: OldScalar).isScalar()) {
848 if (MRI.getType(Reg: OldScalar).getSizeInBits() < sXLen.getSizeInBits()) {
849 Helper.Observer.changingInstr(MI);
850 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: II->ScalarOperand + 2,
851 ExtOpcode: TargetOpcode::G_ANYEXT);
852 Helper.Observer.changedInstr(MI);
853 } else if (MRI.getType(Reg: OldScalar).getSizeInBits() >
854 sXLen.getSizeInBits()) {
855 // TODO: i64 in riscv32.
856 return false;
857 }
858 }
859 }
860 return true;
861 }
862
863 switch (IntrinsicID) {
864 default:
865 return false;
866 case Intrinsic::riscv_clmulh:
867 Helper.MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULH, DstOps: {MI.getOperand(i: 0)},
868 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
869 MI.eraseFromParent();
870 return true;
871 case Intrinsic::vacopy: {
872 // vacopy arguments must be legal because of the intrinsic signature.
873 // No need to check here.
874
875 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
876 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
877 MachineFunction &MF = *MI.getMF();
878 const DataLayout &DL = MIRBuilder.getDataLayout();
879 LLVMContext &Ctx = MF.getFunction().getContext();
880
881 Register DstLst = MI.getOperand(i: 1).getReg();
882 LLT PtrTy = MRI.getType(Reg: DstLst);
883
884 // Load the source va_list
885 Align Alignment = DL.getABITypeAlign(Ty: getTypeForLLT(Ty: PtrTy, C&: Ctx));
886 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
887 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad, MemTy: PtrTy, BaseAlignment: Alignment);
888 auto Tmp = MIRBuilder.buildLoad(Res: PtrTy, Addr: MI.getOperand(i: 2), MMO&: *LoadMMO);
889
890 // Store the result in the destination va_list
891 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
892 PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOStore, MemTy: PtrTy, BaseAlignment: Alignment);
893 MIRBuilder.buildStore(Val: Tmp, Addr: DstLst, MMO&: *StoreMMO);
894
895 MI.eraseFromParent();
896 return true;
897 }
898 case Intrinsic::riscv_vsetvli:
899 case Intrinsic::riscv_vsetvlimax:
900 case Intrinsic::riscv_masked_atomicrmw_add:
901 case Intrinsic::riscv_masked_atomicrmw_sub:
902 case Intrinsic::riscv_masked_atomicrmw_xchg:
903 case Intrinsic::riscv_masked_atomicrmw_max:
904 case Intrinsic::riscv_masked_atomicrmw_min:
905 case Intrinsic::riscv_masked_atomicrmw_umax:
906 case Intrinsic::riscv_masked_atomicrmw_umin:
907 case Intrinsic::riscv_masked_cmpxchg:
908 return true;
909 }
910}
911
912bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
913 MachineIRBuilder &MIRBuilder) const {
914 // Stores the address of the VarArgsFrameIndex slot into the memory location
915 assert(MI.getOpcode() == TargetOpcode::G_VASTART);
916 MachineFunction *MF = MI.getParent()->getParent();
917 RISCVMachineFunctionInfo *FuncInfo = MF->getInfo<RISCVMachineFunctionInfo>();
918 int FI = FuncInfo->getVarArgsFrameIndex();
919 LLT AddrTy = MIRBuilder.getMRI()->getType(Reg: MI.getOperand(i: 0).getReg());
920 auto FINAddr = MIRBuilder.buildFrameIndex(Res: AddrTy, Idx: FI);
921 assert(MI.hasOneMemOperand());
922 MIRBuilder.buildStore(Val: FINAddr, Addr: MI.getOperand(i: 0).getReg(),
923 MMO&: *MI.memoperands()[0]);
924 MI.eraseFromParent();
925 return true;
926}
927
928bool RISCVLegalizerInfo::legalizeReadCounter(
929 MachineInstr &MI, MachineIRBuilder &MIRBuilder,
930 GISelChangeObserver &Observer) const {
931 assert((MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER ||
932 MI.getOpcode() == TargetOpcode::G_READSTEADYCOUNTER) &&
933 "Unexpected opcode");
934 assert(!STI.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
935 "has custom type legalization on riscv32");
936
937 // On RV32 a 64-bit counter CSR must be read as two 32-bit halves. Because
938 // the count may wrap between the two reads, re-read the high half and loop
939 // until the two high reads agree.
940 int64_t LoCounter, HiCounter;
941 if (MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER) {
942 LoCounter = RISCVSysReg::cycle;
943 HiCounter = RISCVSysReg::cycleh;
944 } else {
945 LoCounter = RISCVSysReg::time;
946 HiCounter = RISCVSysReg::timeh;
947 }
948
949 MachineBasicBlock *BB = MI.getParent();
950 MachineFunction &MF = *BB->getParent();
951 const BasicBlock *LLVMBB = BB->getBasicBlock();
952 DebugLoc DL = MI.getDebugLoc();
953 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
954
955 // Split BB into an entry that falls through into a loop block, and a done
956 // block that receives the remainder of BB and its original successors.
957 MachineFunction::iterator It = std::next(x: BB->getIterator());
958 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
959 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
960 MF.insert(MBBI: It, MBB: LoopMBB);
961 MF.insert(MBBI: It, MBB: DoneMBB);
962
963 // Splice the instructions after the readcyclecounter into DoneMBB, notifying
964 // the observer about each moved instruction so CSEInfo stays consistent.
965 for (MachineBasicBlock::iterator I = std::next(x: MI.getIterator()),
966 E = BB->end();
967 I != E; ++I)
968 Observer.changingInstr(MI&: *I);
969 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
970 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
971 for (MachineInstr &MovedMI : DoneMBB->instrs())
972 Observer.changedInstr(MI&: MovedMI);
973 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
974 BB->addSuccessor(Succ: LoopMBB);
975
976 LLT S32 = LLT::scalar(SizeInBits: 32);
977 // Generic vregs carry the s32 type for G_MERGE_VALUES below, but are also
978 // constrained to GPR so the target CSRRS/BNE instructions satisfy the
979 // verifier's register-class constraints.
980 auto CreateGPR = [&]() {
981 Register R = MRI.createGenericVirtualRegister(Ty: S32);
982 MRI.setRegClass(Reg: R, RC: &RISCV::GPRRegClass);
983 return R;
984 };
985 Register LoReg = CreateGPR();
986 Register HiReg = CreateGPR();
987 Register ReadAgainReg = CreateGPR();
988
989 // read:
990 // csrrs HiReg, counterh # high word
991 // csrrs LoReg, counter # low word
992 // csrrs ReadAgainReg, counterh
993 // bne HiReg, ReadAgainReg, read
994 // Emit the target instructions directly with BuildMI.
995 const RISCVInstrInfo *TII = STI.getInstrInfo();
996 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
997 .addImm(Val: HiCounter)
998 .addReg(RegNo: RISCV::X0);
999 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
1000 .addImm(Val: LoCounter)
1001 .addReg(RegNo: RISCV::X0);
1002 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
1003 .addImm(Val: HiCounter)
1004 .addReg(RegNo: RISCV::X0);
1005
1006 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
1007 .addReg(RegNo: HiReg)
1008 .addReg(RegNo: ReadAgainReg)
1009 .addMBB(MBB: LoopMBB);
1010
1011 LoopMBB->addSuccessor(Succ: LoopMBB);
1012 LoopMBB->addSuccessor(Succ: DoneMBB);
1013
1014 // Re-pair the two halves into the 64-bit result.
1015 Register DstReg = MI.getOperand(i: 0).getReg();
1016 Observer.erasingInstr(MI);
1017 MI.eraseFromParent();
1018
1019 MIRBuilder.setInsertPt(MBB&: *DoneMBB, II: DoneMBB->begin());
1020 MIRBuilder.setDebugLoc(DL);
1021 MIRBuilder.buildMergeValues(Res: DstReg, Ops: {LoReg, HiReg});
1022 return true;
1023}
1024
1025bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,
1026 MachineIRBuilder &MIRBuilder) const {
1027 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1028 auto &MF = *MI.getParent()->getParent();
1029 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
1030 unsigned EntrySize = MJTI->getEntrySize(TD: MF.getDataLayout());
1031
1032 Register PtrReg = MI.getOperand(i: 0).getReg();
1033 LLT PtrTy = MRI.getType(Reg: PtrReg);
1034 Register IndexReg = MI.getOperand(i: 2).getReg();
1035 LLT IndexTy = MRI.getType(Reg: IndexReg);
1036
1037 if (!isPowerOf2_32(Value: EntrySize))
1038 return false;
1039
1040 auto ShiftAmt = MIRBuilder.buildConstant(Res: IndexTy, Val: Log2_32(Value: EntrySize));
1041 IndexReg = MIRBuilder.buildShl(Dst: IndexTy, Src0: IndexReg, Src1: ShiftAmt).getReg(Idx: 0);
1042
1043 auto Addr = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: IndexReg);
1044
1045 MachineMemOperand *MMO = MF.getMachineMemOperand(
1046 PtrInfo: MachinePointerInfo::getJumpTable(MF), F: MachineMemOperand::MOLoad,
1047 Size: EntrySize, BaseAlignment: Align(MJTI->getEntryAlignment(TD: MF.getDataLayout())));
1048
1049 Register TargetReg;
1050 switch (MJTI->getEntryKind()) {
1051 default:
1052 return false;
1053 case MachineJumpTableInfo::EK_LabelDifference32: {
1054 // For PIC, the sequence is:
1055 // BRIND(load(Jumptable + index) + RelocBase)
1056 // RelocBase can be JumpTable, GOT or some sort of global base.
1057 unsigned LoadOpc =
1058 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;
1059 auto Load = MIRBuilder.buildLoadInstr(Opcode: LoadOpc, Res: IndexTy, Addr, MMO&: *MMO);
1060 TargetReg = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: PtrReg, Op1: Load).getReg(Idx: 0);
1061 break;
1062 }
1063 case MachineJumpTableInfo::EK_Custom32: {
1064 auto Load = MIRBuilder.buildLoadInstr(Opcode: TargetOpcode::G_SEXTLOAD, Res: IndexTy,
1065 Addr, MMO&: *MMO);
1066 TargetReg = MIRBuilder.buildIntToPtr(Dst: PtrTy, Src: Load).getReg(Idx: 0);
1067 break;
1068 }
1069 case MachineJumpTableInfo::EK_BlockAddress:
1070 TargetReg = MIRBuilder.buildLoad(Res: PtrTy, Addr, MMO&: *MMO).getReg(Idx: 0);
1071 break;
1072 }
1073
1074 MIRBuilder.buildBrIndirect(Tgt: TargetReg);
1075
1076 MI.eraseFromParent();
1077 return true;
1078}
1079
1080bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,
1081 bool ShouldOptForSize) const {
1082 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);
1083 int64_t Imm = APImm.getSExtValue();
1084 // All simm32 constants should be handled by isel.
1085 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
1086 // this check redundant, but small immediates are common so this check
1087 // should have better compile time.
1088 if (isInt<32>(x: Imm))
1089 return false;
1090
1091 // We only need to cost the immediate, if constant pool lowering is enabled.
1092 if (!STI.useConstantPoolForLargeInts())
1093 return false;
1094
1095 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI);
1096 if (Seq.size() <= STI.getMaxBuildIntsCost())
1097 return false;
1098
1099 // Optimizations below are disabled for opt size. If we're optimizing for
1100 // size, use a constant pool.
1101 if (ShouldOptForSize)
1102 return true;
1103 //
1104 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
1105 // that if it will avoid a constant pool.
1106 // It will require an extra temporary register though.
1107 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
1108 // low and high 32 bits are the same and bit 31 and 63 are set.
1109 unsigned ShiftAmt, AddOpc;
1110 RISCVMatInt::InstSeq SeqLo =
1111 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI, ShiftAmt, AddOpc);
1112 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
1113}
1114
1115bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
1116 MachineIRBuilder &MIB) const {
1117 Register Dst = MI.getOperand(i: 0).getReg();
1118
1119 // We define our scalable vector types for lmul=1 to use a 64 bit known
1120 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
1121 // vscale as VLENB / 8.
1122 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
1123 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1124 // Support for VLEN==32 is incomplete.
1125 return false;
1126
1127 // We assume VLENB is a multiple of 8. We manually choose the best shift
1128 // here because SimplifyDemandedBits isn't always able to simplify it.
1129 uint64_t Val = MI.getOperand(i: 1).getCImm()->getZExtValue();
1130 if (isPowerOf2_64(Value: Val)) {
1131 uint64_t Log2 = Log2_64(Value: Val);
1132 if (Log2 < 3) {
1133 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1134 MIB.buildLShr(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3 - Log2),
1135 Flags: MachineInstr::IsExact);
1136 } else if (Log2 > 3) {
1137 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1138 MIB.buildShl(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Log2 - 3));
1139 } else {
1140 MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {Dst}, SrcOps: {});
1141 }
1142 } else if ((Val % 8) == 0) {
1143 // If the multiplier is a multiple of 8, scale it down to avoid needing
1144 // to shift the VLENB value.
1145 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1146 MIB.buildMul(Dst, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: Val / 8));
1147 } else {
1148 auto VLENB = MIB.buildInstr(Opc: RISCV::G_READ_VLENB, DstOps: {sXLen}, SrcOps: {});
1149 auto VScale = MIB.buildLShr(Dst: sXLen, Src0: VLENB, Src1: MIB.buildConstant(Res: sXLen, Val: 3),
1150 Flags: MachineInstr::IsExact);
1151 MIB.buildMul(Dst, Src0: VScale, Src1: MIB.buildConstant(Res: sXLen, Val));
1152 }
1153 MI.eraseFromParent();
1154 return true;
1155}
1156
1157// Custom-lower extensions from mask vectors by using a vselect either with 1
1158// for zero/any-extension or -1 for sign-extension:
1159// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
1160// Note that any-extension is lowered identically to zero-extension.
1161bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,
1162 MachineIRBuilder &MIB) const {
1163
1164 unsigned Opc = MI.getOpcode();
1165 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||
1166 Opc == TargetOpcode::G_ANYEXT);
1167
1168 MachineRegisterInfo &MRI = *MIB.getMRI();
1169 Register Dst = MI.getOperand(i: 0).getReg();
1170 Register Src = MI.getOperand(i: 1).getReg();
1171
1172 LLT DstTy = MRI.getType(Reg: Dst);
1173 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;
1174 LLT DstEltTy = DstTy.getElementType();
1175 auto SplatZero = MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: 0));
1176 auto SplatTrue =
1177 MIB.buildSplatVector(Res: DstTy, Val: MIB.buildConstant(Res: DstEltTy, Val: ExtTrueVal));
1178 MIB.buildSelect(Res: Dst, Tst: Src, Op0: SplatTrue, Op1: SplatZero);
1179
1180 MI.eraseFromParent();
1181 return true;
1182}
1183
1184bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,
1185 LegalizerHelper &Helper,
1186 MachineIRBuilder &MIB) const {
1187 assert((isa<GLoad>(MI) || isa<GStore>(MI)) &&
1188 "Machine instructions must be Load/Store.");
1189 MachineRegisterInfo &MRI = *MIB.getMRI();
1190 MachineFunction *MF = MI.getMF();
1191 const DataLayout &DL = MIB.getDataLayout();
1192 LLVMContext &Ctx = MF->getFunction().getContext();
1193
1194 Register DstReg = MI.getOperand(i: 0).getReg();
1195 LLT DataTy = MRI.getType(Reg: DstReg);
1196 if (!DataTy.isVector())
1197 return false;
1198
1199 if (!MI.hasOneMemOperand())
1200 return false;
1201
1202 MachineMemOperand *MMO = *MI.memoperands_begin();
1203
1204 const auto *TLI = STI.getTargetLowering();
1205 EVT VT = EVT::getEVT(Ty: getTypeForLLT(Ty: DataTy, C&: Ctx));
1206
1207 if (TLI->allowsMemoryAccessForAlignment(Context&: Ctx, DL, VT, MMO: *MMO))
1208 return true;
1209
1210 unsigned EltSizeBits = DataTy.getScalarSizeInBits();
1211 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
1212 "Unexpected unaligned RVV load type");
1213
1214 // Calculate the new vector type with i8 elements
1215 unsigned NumElements =
1216 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);
1217 LLT NewDataTy = LLT::scalable_vector(MinNumElements: NumElements, ScalarSizeInBits: 8);
1218
1219 Helper.bitcast(MI, TypeIdx: 0, Ty: NewDataTy);
1220
1221 return true;
1222}
1223
1224/// Return the type of the mask type suitable for masking the provided
1225/// vector type. This is simply an i1 element type vector of the same
1226/// (possibly scalable) length.
1227static LLT getMaskTypeFor(LLT VecTy) {
1228 assert(VecTy.isVector());
1229 ElementCount EC = VecTy.getElementCount();
1230 return LLT::vector(EC, ScalarTy: LLT::scalar(SizeInBits: 1));
1231}
1232
1233/// Creates an all ones mask suitable for masking a vector of type VecTy with
1234/// vector length VL.
1235static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL,
1236 MachineIRBuilder &MIB,
1237 MachineRegisterInfo &MRI) {
1238 LLT MaskTy = getMaskTypeFor(VecTy);
1239 return MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {MaskTy}, SrcOps: {VL});
1240}
1241
1242/// Gets the two common "VL" operands: an all-ones mask and the vector length.
1243/// VecTy is a scalable vector type.
1244static std::pair<MachineInstrBuilder, MachineInstrBuilder>
1245buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1246 assert(VecTy.isScalableVector() && "Expecting scalable container type");
1247 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
1248 LLT XLenTy(STI.getXLenVT());
1249 auto VL = MIB.buildConstant(Res: XLenTy, Val: -1);
1250 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
1251 return {Mask, VL};
1252}
1253
1254static MachineInstrBuilder
1255buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
1256 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
1257 MachineRegisterInfo &MRI) {
1258 // TODO: If the Hi bits of the splat are undefined, then it's fine to just
1259 // splat Lo even if it might be sign extended. I don't think we have
1260 // introduced a case where we're build a s64 where the upper bits are undef
1261 // yet.
1262
1263 // Fall back to a stack store and stride x0 vector load.
1264 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in
1265 // preprocessDAG in SDAG.
1266 return MIB.buildInstr(Opc: RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, DstOps: {Dst},
1267 SrcOps: {Passthru, Lo, Hi, VL});
1268}
1269
1270static MachineInstrBuilder
1271buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
1272 const SrcOp &Scalar, const SrcOp &VL,
1273 MachineIRBuilder &MIB, MachineRegisterInfo &MRI) {
1274 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
1275 auto Unmerge = MIB.buildUnmerge(Res: LLT::scalar(SizeInBits: 32), Op: Scalar);
1276 return buildSplatPartsS64WithVL(Dst, Passthru, Lo: Unmerge.getReg(Idx: 0),
1277 Hi: Unmerge.getReg(Idx: 1), VL, MIB, MRI);
1278}
1279
1280// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a
1281// legal equivalently-sized i8 type, so we can use that as a go-between.
1282// Splats of s1 types that have constant value can be legalized as VMSET_VL or
1283// VMCLR_VL.
1284bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
1285 MachineIRBuilder &MIB) const {
1286 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);
1287
1288 MachineRegisterInfo &MRI = *MIB.getMRI();
1289
1290 Register Dst = MI.getOperand(i: 0).getReg();
1291 Register SplatVal = MI.getOperand(i: 1).getReg();
1292
1293 LLT VecTy = MRI.getType(Reg: Dst);
1294 LLT XLenTy(STI.getXLenVT());
1295
1296 // Handle case of s64 element vectors on rv32
1297 if (XLenTy.getSizeInBits() == 32 &&
1298 VecTy.getElementType().getSizeInBits() == 64) {
1299 auto [_, VL] = buildDefaultVLOps(VecTy: MRI.getType(Reg: Dst), MIB, MRI);
1300 buildSplatSplitS64WithVL(Dst, Passthru: MIB.buildUndef(Res: VecTy), Scalar: SplatVal, VL, MIB,
1301 MRI);
1302 MI.eraseFromParent();
1303 return true;
1304 }
1305
1306 // All-zeros or all-ones splats are handled specially.
1307 MachineInstr &SplatValMI = *MRI.getVRegDef(Reg: SplatVal);
1308 if (isAllOnesOrAllOnesSplat(MI: SplatValMI, MRI)) {
1309 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1310 MIB.buildInstr(Opc: RISCV::G_VMSET_VL, DstOps: {Dst}, SrcOps: {VL});
1311 MI.eraseFromParent();
1312 return true;
1313 }
1314 if (isNullOrNullSplat(MI: SplatValMI, MRI)) {
1315 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1316 MIB.buildInstr(Opc: RISCV::G_VMCLR_VL, DstOps: {Dst}, SrcOps: {VL});
1317 MI.eraseFromParent();
1318 return true;
1319 }
1320
1321 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all
1322 // ones) by promoting it to an s8 splat.
1323 LLT InterEltTy = LLT::scalar(SizeInBits: 8);
1324 LLT InterTy = VecTy.changeElementType(NewEltTy: InterEltTy);
1325 auto ZExtSplatVal = MIB.buildZExt(Res: InterEltTy, Op: SplatVal);
1326 auto And =
1327 MIB.buildAnd(Dst: InterEltTy, Src0: ZExtSplatVal, Src1: MIB.buildConstant(Res: InterEltTy, Val: 1));
1328 auto LHS = MIB.buildSplatVector(Res: InterTy, Val: And);
1329 auto ZeroSplat =
1330 MIB.buildSplatVector(Res: InterTy, Val: MIB.buildConstant(Res: InterEltTy, Val: 0));
1331 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: LHS, Op1: ZeroSplat);
1332 MI.eraseFromParent();
1333 return true;
1334}
1335
1336static LLT getLMUL1Ty(LLT VecTy) {
1337 assert(VecTy.getElementType().getSizeInBits() <= 64 &&
1338 "Unexpected vector LLT");
1339 return LLT::scalable_vector(MinNumElements: RISCV::RVVBitsPerBlock /
1340 VecTy.getElementType().getSizeInBits(),
1341 ScalarTy: VecTy.getElementType());
1342}
1343
1344bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
1345 MachineIRBuilder &MIB) const {
1346 GExtractSubvector &ES = cast<GExtractSubvector>(Val&: MI);
1347
1348 MachineRegisterInfo &MRI = *MIB.getMRI();
1349
1350 Register Dst = ES.getReg(Idx: 0);
1351 Register Src = ES.getSrcVec();
1352 uint64_t Idx = ES.getIndexImm();
1353
1354 // With an index of 0 this is a cast-like subvector, which can be performed
1355 // with subregister operations.
1356 if (Idx == 0)
1357 return true;
1358
1359 LLT LitTy = MRI.getType(Reg: Dst);
1360 LLT BigTy = MRI.getType(Reg: Src);
1361
1362 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1363 // We can't slide this mask vector up indexed by its i1 elements.
1364 // This poses a problem when we wish to insert a scalable vector which
1365 // can't be re-expressed as a larger type. Just choose the slow path and
1366 // extend to a larger type, then truncate back down.
1367 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1368 LLT ExtLitTy = LitTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1369 auto BigZExt = MIB.buildZExt(Res: ExtBigTy, Op: Src);
1370 auto ExtractZExt = MIB.buildExtractSubvector(Res: ExtLitTy, Src: BigZExt, Index: Idx);
1371 auto SplatZero = MIB.buildSplatVector(
1372 Res: ExtLitTy, Val: MIB.buildConstant(Res: ExtLitTy.getElementType(), Val: 0));
1373 MIB.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: Dst, Op0: ExtractZExt, Op1: SplatZero);
1374 MI.eraseFromParent();
1375 return true;
1376 }
1377
1378 // extract_subvector scales the index by vscale if the subvector is scalable,
1379 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
1380 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1381 MVT LitTyMVT = getMVTForLLT(Ty: LitTy);
1382 auto Decompose =
1383 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1384 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: LitTyMVT, InsertExtractIdx: Idx, TRI);
1385 unsigned RemIdx = Decompose.second;
1386
1387 // If the Idx has been completely eliminated then this is a subvector extract
1388 // which naturally aligns to a vector register. These can easily be handled
1389 // using subregister manipulation.
1390 if (RemIdx == 0)
1391 return true;
1392
1393 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
1394 // was > M1 then the index would need to be a multiple of VLMAX, and so would
1395 // divide exactly.
1396 assert(
1397 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(LitTyMVT)).second ||
1398 RISCVTargetLowering::getLMUL(LitTyMVT) == RISCVVType::LMUL_1);
1399
1400 // If the vector type is an LMUL-group type, extract a subvector equal to the
1401 // nearest full vector register type.
1402 LLT InterLitTy = BigTy;
1403 Register Vec = Src;
1404 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1405 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1406 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
1407 // we should have successfully decomposed the extract into a subregister.
1408 assert(Decompose.first != RISCV::NoSubRegister);
1409 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1410 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
1411 // specified on the source Register (the equivalent) since generic virtual
1412 // register does not allow subregister index.
1413 Vec = MIB.buildExtractSubvector(Res: InterLitTy, Src, Index: Idx - RemIdx).getReg(Idx: 0);
1414 }
1415
1416 // Slide this vector register down by the desired number of elements in order
1417 // to place the desired subvector starting at element 0.
1418 const LLT XLenTy(STI.getXLenVT());
1419 auto SlidedownAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1420 auto [Mask, VL] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1421 uint64_t Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
1422 auto Slidedown = MIB.buildInstr(
1423 Opc: RISCV::G_VSLIDEDOWN_VL, DstOps: {InterLitTy},
1424 SrcOps: {MIB.buildUndef(Res: InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
1425
1426 // Now the vector is in the right position, extract our final subvector. This
1427 // should resolve to a COPY.
1428 MIB.buildExtractSubvector(Res: Dst, Src: Slidedown, Index: 0);
1429
1430 MI.eraseFromParent();
1431 return true;
1432}
1433
1434bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,
1435 LegalizerHelper &Helper,
1436 MachineIRBuilder &MIB) const {
1437 GInsertSubvector &IS = cast<GInsertSubvector>(Val&: MI);
1438
1439 MachineRegisterInfo &MRI = *MIB.getMRI();
1440
1441 Register Dst = IS.getReg(Idx: 0);
1442 Register BigVec = IS.getBigVec();
1443 Register LitVec = IS.getSubVec();
1444 uint64_t Idx = IS.getIndexImm();
1445
1446 LLT BigTy = MRI.getType(Reg: BigVec);
1447 LLT LitTy = MRI.getType(Reg: LitVec);
1448
1449 if (Idx == 0 && mi_match(R: BigVec, MRI, P: m_GImplicitDef()))
1450 return true;
1451
1452 // We don't have the ability to slide mask vectors up indexed by their i1
1453 // elements; the smallest we can do is i8. Often we are able to bitcast to
1454 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i8
1455 // vectors and truncate down after the insert.
1456 if (LitTy.getElementType() == LLT::scalar(SizeInBits: 1)) {
1457 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();
1458 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();
1459 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)
1460 return Helper.bitcast(
1461 MI&: IS, TypeIdx: 0,
1462 Ty: LLT::vector(EC: BigTy.getElementCount().divideCoefficientBy(RHS: 8), ScalarSizeInBits: 8));
1463
1464 // We can't slide this mask vector up indexed by its i1 elements.
1465 // This poses a problem when we wish to insert a scalable vector which
1466 // can't be re-expressed as a larger type. Just choose the slow path and
1467 // extend to a larger type, then truncate back down.
1468 LLT ExtBigTy = BigTy.changeElementType(NewEltTy: LLT::scalar(SizeInBits: 8));
1469 return Helper.widenScalar(MI&: IS, TypeIdx: 0, WideTy: ExtBigTy);
1470 }
1471
1472 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1473 unsigned SubRegIdx, RemIdx;
1474 std::tie(args&: SubRegIdx, args&: RemIdx) =
1475 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1476 VecVT: getMVTForLLT(Ty: BigTy), SubVecVT: getMVTForLLT(Ty: LitTy), InsertExtractIdx: Idx, TRI);
1477
1478 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
1479 assert(isPowerOf2_64(
1480 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));
1481 bool ExactlyVecRegSized =
1482 STI.expandVScale(X: LitTy.getSizeInBits())
1483 .isKnownMultipleOf(RHS: STI.expandVScale(X: VecRegSize));
1484
1485 // If the Idx has been completely eliminated and this subvector's size is a
1486 // vector register or a multiple thereof, or the surrounding elements are
1487 // undef, then this is a subvector insert which naturally aligns to a vector
1488 // register. These can easily be handled using subregister manipulation.
1489 if (RemIdx == 0 && ExactlyVecRegSized)
1490 return true;
1491
1492 // If the subvector is smaller than a vector register, then the insertion
1493 // must preserve the undisturbed elements of the register. We do this by
1494 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
1495 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
1496 // subvector within the vector register, and an INSERT_SUBVECTOR of that
1497 // LMUL=1 type back into the larger vector (resolving to another subregister
1498 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
1499 // to avoid allocating a large register group to hold our subvector.
1500
1501 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
1502 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
1503 // (in our case undisturbed). This means we can set up a subvector insertion
1504 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
1505 // size of the subvector.
1506 const LLT XLenTy(STI.getXLenVT());
1507 LLT InterLitTy = BigTy;
1508 Register AlignedExtract = BigVec;
1509 unsigned AlignedIdx = Idx - RemIdx;
1510 if (TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(),
1511 RHS: getLMUL1Ty(VecTy: BigTy).getSizeInBits())) {
1512 InterLitTy = getLMUL1Ty(VecTy: BigTy);
1513 // Extract a subvector equal to the nearest full vector register type. This
1514 // should resolve to a G_EXTRACT on a subreg.
1515 AlignedExtract =
1516 MIB.buildExtractSubvector(Res: InterLitTy, Src: BigVec, Index: AlignedIdx).getReg(Idx: 0);
1517 }
1518
1519 auto Insert = MIB.buildInsertSubvector(Res: InterLitTy, Src0: MIB.buildUndef(Res: InterLitTy),
1520 Src1: LitVec, Index: 0);
1521
1522 auto [Mask, _] = buildDefaultVLOps(VecTy: InterLitTy, MIB, MRI);
1523 auto VL = MIB.buildVScale(Res: XLenTy, MinElts: LitTy.getElementCount().getKnownMinValue());
1524
1525 // If we're inserting into the lowest elements, use a tail undisturbed
1526 // vmv.v.v.
1527 MachineInstrBuilder Inserted;
1528 bool NeedInsertSubvec =
1529 TypeSize::isKnownGT(LHS: BigTy.getSizeInBits(), RHS: InterLitTy.getSizeInBits());
1530 Register InsertedDst =
1531 NeedInsertSubvec ? MRI.createGenericVirtualRegister(Ty: InterLitTy) : Dst;
1532 if (RemIdx == 0) {
1533 Inserted = MIB.buildInstr(Opc: RISCV::G_VMV_V_V_VL, DstOps: {InsertedDst},
1534 SrcOps: {AlignedExtract, Insert, VL});
1535 } else {
1536 auto SlideupAmt = MIB.buildVScale(Res: XLenTy, MinElts: RemIdx);
1537 // Construct the vector length corresponding to RemIdx + length(LitTy).
1538 VL = MIB.buildAdd(Dst: XLenTy, Src0: SlideupAmt, Src1: VL);
1539 // Use tail agnostic policy if we're inserting over InterLitTy's tail.
1540 ElementCount EndIndex =
1541 ElementCount::getScalable(MinVal: RemIdx) + LitTy.getElementCount();
1542 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
1543 if (STI.expandVScale(X: EndIndex) ==
1544 STI.expandVScale(X: InterLitTy.getElementCount()))
1545 Policy = RISCVVType::TAIL_AGNOSTIC;
1546
1547 Inserted =
1548 MIB.buildInstr(Opc: RISCV::G_VSLIDEUP_VL, DstOps: {InsertedDst},
1549 SrcOps: {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});
1550 }
1551
1552 // If required, insert this subvector back into the correct vector register.
1553 // This should resolve to an INSERT_SUBREG instruction.
1554 if (NeedInsertSubvec)
1555 MIB.buildInsertSubvector(Res: Dst, Src0: BigVec, Src1: Inserted, Index: AlignedIdx);
1556
1557 MI.eraseFromParent();
1558 return true;
1559}
1560
1561bool RISCVLegalizerInfo::legalizeBitreverse(MachineInstr &MI,
1562 MachineIRBuilder &MIB) const {
1563 assert(MI.getOpcode() == TargetOpcode::G_BITREVERSE && "Unexpected opcode");
1564
1565 if (!STI.hasStdExtZbkb())
1566 return false;
1567
1568 MachineRegisterInfo &MRI = *MIB.getMRI();
1569
1570 Register Dst = MI.getOperand(i: 0).getReg();
1571 Register Src = MI.getOperand(i: 1).getReg();
1572
1573 if (!MRI.getType(Reg: Dst).isScalar(Size: 8))
1574 return false;
1575
1576 auto WideSrc = MIB.buildAnyExt(Res: sXLen, Op: Src);
1577 auto Brev = MIB.buildInstr(Opc: RISCV::G_BREV8, DstOps: {sXLen}, SrcOps: {WideSrc.getReg(Idx: 0)});
1578 MIB.buildTrunc(Res: Dst, Op: Brev.getReg(Idx: 0));
1579
1580 MI.eraseFromParent();
1581 return true;
1582}
1583
1584static unsigned getRISCVWOpcode(unsigned Opcode) {
1585 switch (Opcode) {
1586 default:
1587 llvm_unreachable("Unexpected opcode");
1588 case TargetOpcode::G_ASHR:
1589 return RISCV::G_SRAW;
1590 case TargetOpcode::G_LSHR:
1591 return RISCV::G_SRLW;
1592 case TargetOpcode::G_SHL:
1593 return RISCV::G_SLLW;
1594 case TargetOpcode::G_SDIV:
1595 return RISCV::G_DIVW;
1596 case TargetOpcode::G_UDIV:
1597 return RISCV::G_DIVUW;
1598 case TargetOpcode::G_UREM:
1599 return RISCV::G_REMUW;
1600 case TargetOpcode::G_ROTL:
1601 return RISCV::G_ROLW;
1602 case TargetOpcode::G_ROTR:
1603 return RISCV::G_RORW;
1604 case TargetOpcode::G_CTLZ:
1605 return RISCV::G_CLZW;
1606 case TargetOpcode::G_CTTZ:
1607 return RISCV::G_CTZW;
1608 case TargetOpcode::G_CTLS:
1609 return RISCV::G_CLSW;
1610 case TargetOpcode::G_FPTOSI:
1611 return RISCV::G_FCVT_W_RV64;
1612 case TargetOpcode::G_FPTOUI:
1613 return RISCV::G_FCVT_WU_RV64;
1614 }
1615}
1616
1617bool RISCVLegalizerInfo::legalizeCustom(
1618 LegalizerHelper &Helper, MachineInstr &MI,
1619 LostDebugLocObserver &LocObserver) const {
1620 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1621 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1622 MachineFunction &MF = *MI.getParent()->getParent();
1623 switch (MI.getOpcode()) {
1624 default:
1625 // No idea what to do.
1626 return false;
1627 case TargetOpcode::G_ABS:
1628 return Helper.lowerAbsToMaxNeg(MI);
1629 case TargetOpcode::G_CLMULH: {
1630 assert(STI.is64Bit() &&
1631 MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32) &&
1632 "Unexpected custom legalization");
1633 // Shift both inputs by 32 so the full product has 64 trailing zeros.
1634 // CLMULH then returns the original 64-bit product. Extract its high half.
1635 auto Shift = MIRBuilder.buildConstant(Res: sXLen, Val: 32);
1636 auto LHS = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 1));
1637 auto RHS = MIRBuilder.buildAnyExt(Res: sXLen, Op: MI.getOperand(i: 2));
1638 auto ShiftedLHS = MIRBuilder.buildShl(Dst: sXLen, Src0: LHS, Src1: Shift);
1639 auto ShiftedRHS = MIRBuilder.buildShl(Dst: sXLen, Src0: RHS, Src1: Shift);
1640 auto Product = MIRBuilder.buildInstr(Opc: TargetOpcode::G_CLMULH, DstOps: {sXLen},
1641 SrcOps: {ShiftedLHS, ShiftedRHS});
1642 auto High = MIRBuilder.buildLShr(Dst: sXLen, Src0: Product, Src1: Shift);
1643 MIRBuilder.buildTrunc(Res: MI.getOperand(i: 0), Op: High);
1644 MI.eraseFromParent();
1645 return true;
1646 }
1647 case TargetOpcode::G_FCONSTANT: {
1648 const APFloat &FVal = MI.getOperand(i: 1).getFPImm()->getValueAPF();
1649
1650 // Convert G_FCONSTANT to G_CONSTANT.
1651 Register DstReg = MI.getOperand(i: 0).getReg();
1652 MIRBuilder.buildConstant(Res: DstReg, Val: FVal.bitcastToAPInt());
1653
1654 MI.eraseFromParent();
1655 return true;
1656 }
1657 case TargetOpcode::G_CONSTANT: {
1658 const Function &F = MF.getFunction();
1659 // TODO: if PSI and BFI are present, add " ||
1660 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".
1661 bool ShouldOptForSize = F.hasOptSize();
1662 const ConstantInt *ConstVal = MI.getOperand(i: 1).getCImm();
1663 if (!shouldBeInConstantPool(APImm: ConstVal->getValue(), ShouldOptForSize))
1664 return true;
1665 return Helper.lowerConstant(MI);
1666 }
1667 case TargetOpcode::G_SUB:
1668 case TargetOpcode::G_ADD: {
1669 Helper.Observer.changingInstr(MI);
1670 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1671 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1672
1673 Register DstALU = MRI.createGenericVirtualRegister(Ty: sXLen);
1674
1675 MachineOperand &MO = MI.getOperand(i: 0);
1676 MIRBuilder.setInsertPt(MBB&: MIRBuilder.getMBB(), II: ++MIRBuilder.getInsertPt());
1677 auto DstSext = MIRBuilder.buildSExtInReg(Res: sXLen, Op: DstALU, ImmOp: 32);
1678
1679 MIRBuilder.buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: {MO}, SrcOps: {DstSext});
1680 MO.setReg(DstALU);
1681
1682 Helper.Observer.changedInstr(MI);
1683 return true;
1684 }
1685 case TargetOpcode::G_ASHR:
1686 case TargetOpcode::G_LSHR:
1687 case TargetOpcode::G_SHL: {
1688 if (getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI)) {
1689 // We don't need a custom node for shift by constant. Just widen the
1690 // source and the shift amount.
1691 unsigned ExtOpc = TargetOpcode::G_ANYEXT;
1692 if (MI.getOpcode() == TargetOpcode::G_ASHR)
1693 ExtOpc = TargetOpcode::G_SEXT;
1694 else if (MI.getOpcode() == TargetOpcode::G_LSHR)
1695 ExtOpc = TargetOpcode::G_ZEXT;
1696
1697 Helper.Observer.changingInstr(MI);
1698 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: ExtOpc);
1699 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ZEXT);
1700 Helper.widenScalarDst(MI, WideTy: sXLen);
1701 Helper.Observer.changedInstr(MI);
1702 return true;
1703 }
1704
1705 Helper.Observer.changingInstr(MI);
1706 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1707 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1708 Helper.widenScalarDst(MI, WideTy: sXLen);
1709 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1710 Helper.Observer.changedInstr(MI);
1711 return true;
1712 }
1713 case TargetOpcode::G_SDIV:
1714 case TargetOpcode::G_UDIV:
1715 case TargetOpcode::G_UREM:
1716 case TargetOpcode::G_ROTL:
1717 case TargetOpcode::G_ROTR: {
1718 Helper.Observer.changingInstr(MI);
1719 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1720 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 2, ExtOpcode: TargetOpcode::G_ANYEXT);
1721 Helper.widenScalarDst(MI, WideTy: sXLen);
1722 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1723 Helper.Observer.changedInstr(MI);
1724 return true;
1725 }
1726 case TargetOpcode::G_CTLZ:
1727 case TargetOpcode::G_CTTZ:
1728 case TargetOpcode::G_CTLS: {
1729 Helper.Observer.changingInstr(MI);
1730 Helper.widenScalarSrc(MI, WideTy: sXLen, OpIdx: 1, ExtOpcode: TargetOpcode::G_ANYEXT);
1731 Helper.widenScalarDst(MI, WideTy: sXLen);
1732 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1733 Helper.Observer.changedInstr(MI);
1734 return true;
1735 }
1736 case TargetOpcode::G_FPTOSI:
1737 case TargetOpcode::G_FPTOUI: {
1738 Helper.Observer.changingInstr(MI);
1739 Helper.widenScalarDst(MI, WideTy: sXLen);
1740 MI.setDesc(MIRBuilder.getTII().get(Opcode: getRISCVWOpcode(Opcode: MI.getOpcode())));
1741 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RTZ));
1742 Helper.Observer.changedInstr(MI);
1743 return true;
1744 }
1745 case TargetOpcode::G_LROUND: {
1746 // The (i32 any_lround) Pat is IsRV32-only; on RV64 lower to
1747 // riscv_fcvt_w_rv64 with FRM_RMM.
1748 Helper.Observer.changingInstr(MI);
1749 Helper.widenScalarDst(MI, WideTy: sXLen);
1750 MI.setDesc(MIRBuilder.getTII().get(Opcode: RISCV::G_FCVT_W_RV64));
1751 MI.addOperand(Op: MachineOperand::CreateImm(Val: RISCVFPRndMode::RMM));
1752 Helper.Observer.changedInstr(MI);
1753 return true;
1754 }
1755 case TargetOpcode::G_READCYCLECOUNTER:
1756 case TargetOpcode::G_READSTEADYCOUNTER:
1757 return legalizeReadCounter(MI, MIRBuilder, Observer&: Helper.Observer);
1758 case TargetOpcode::G_IS_FPCLASS: {
1759 Register GISFPCLASS = MI.getOperand(i: 0).getReg();
1760 Register Src = MI.getOperand(i: 1).getReg();
1761 const MachineOperand &ImmOp = MI.getOperand(i: 2);
1762 MachineIRBuilder MIB(MI);
1763
1764 // Turn LLVM IR's floating point classes to that in RISC-V,
1765 // by simply rotating the 10-bit immediate right by two bits.
1766 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));
1767 auto FClassMask = MIB.buildConstant(Res: sXLen, Val: GFpClassImm.rotr(rotateAmt: 2).zext(width: XLen));
1768 auto ConstZero = MIB.buildConstant(Res: sXLen, Val: 0);
1769
1770 auto GFClass = MIB.buildInstr(Opc: RISCV::G_FCLASS, DstOps: {sXLen}, SrcOps: {Src});
1771 auto And = MIB.buildAnd(Dst: sXLen, Src0: GFClass, Src1: FClassMask);
1772 MIB.buildICmp(Pred: CmpInst::ICMP_NE, Res: GISFPCLASS, Op0: And, Op1: ConstZero);
1773
1774 MI.eraseFromParent();
1775 return true;
1776 }
1777 case TargetOpcode::G_BRJT:
1778 return legalizeBRJT(MI, MIRBuilder);
1779 case TargetOpcode::G_VASTART:
1780 return legalizeVAStart(MI, MIRBuilder);
1781 case TargetOpcode::G_VSCALE:
1782 return legalizeVScale(MI, MIB&: MIRBuilder);
1783 case TargetOpcode::G_ZEXT:
1784 case TargetOpcode::G_SEXT:
1785 case TargetOpcode::G_ANYEXT:
1786 return legalizeExt(MI, MIB&: MIRBuilder);
1787 case TargetOpcode::G_SPLAT_VECTOR:
1788 return legalizeSplatVector(MI, MIB&: MIRBuilder);
1789 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1790 return legalizeExtractSubvector(MI, MIB&: MIRBuilder);
1791 case TargetOpcode::G_INSERT_SUBVECTOR:
1792 return legalizeInsertSubvector(MI, Helper, MIB&: MIRBuilder);
1793 case TargetOpcode::G_BITREVERSE:
1794 return legalizeBitreverse(MI, MIB&: MIRBuilder);
1795 case TargetOpcode::G_LOAD:
1796 case TargetOpcode::G_STORE:
1797 return legalizeLoadStore(MI, Helper, MIB&: MIRBuilder);
1798 }
1799
1800 llvm_unreachable("expected switch to return");
1801}
1802