1//===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64LegalizerInfo.h"
15#include "AArch64Subtarget.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetOpcodes.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/IntrinsicsAArch64.h"
30#include "llvm/IR/Type.h"
31#include "llvm/Support/MathExtras.h"
32#include <initializer_list>
33
34#define DEBUG_TYPE "aarch64-legalinfo"
35
36using namespace llvm;
37using namespace LegalizeActions;
38using namespace LegalizeMutations;
39using namespace LegalityPredicates;
40using namespace MIPatternMatch;
41
42AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
43 : ST(&ST) {
44 using namespace TargetOpcode;
45 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
46 const LLT s8 = LLT::scalar(SizeInBits: 8);
47 const LLT s16 = LLT::scalar(SizeInBits: 16);
48 const LLT s32 = LLT::scalar(SizeInBits: 32);
49 const LLT s64 = LLT::scalar(SizeInBits: 64);
50 const LLT s128 = LLT::scalar(SizeInBits: 128);
51 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
52 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
53 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
54 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
55 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
56 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
57 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
58 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
59 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
60 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
61 const LLT v2p0 = LLT::fixed_vector(NumElements: 2, ScalarTy: p0);
62
63 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
64 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
65 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
66 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
67
68 const LLT bf16 = LLT::bfloat16();
69 const LLT v4bf16 = LLT::fixed_vector(NumElements: 4, ScalarTy: bf16);
70 const LLT v8bf16 = LLT::fixed_vector(NumElements: 8, ScalarTy: bf16);
71
72 const LLT f16 = LLT::float16();
73 const LLT v4f16 = LLT::fixed_vector(NumElements: 4, ScalarTy: f16);
74 const LLT v8f16 = LLT::fixed_vector(NumElements: 8, ScalarTy: f16);
75
76 const LLT f32 = LLT::float32();
77 const LLT v2f32 = LLT::fixed_vector(NumElements: 2, ScalarTy: f32);
78 const LLT v4f32 = LLT::fixed_vector(NumElements: 4, ScalarTy: f32);
79
80 const LLT f64 = LLT::float64();
81 const LLT v2f64 = LLT::fixed_vector(NumElements: 2, ScalarTy: f64);
82
83 const LLT f128 = LLT::float128();
84
85 const LLT i8 = LLT::integer(SizeInBits: 8);
86 const LLT v8i8 = LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
87 const LLT v16i8 = LLT::fixed_vector(NumElements: 16, ScalarTy: i8);
88
89 const LLT i16 = LLT::integer(SizeInBits: 16);
90 const LLT v8i16 = LLT::fixed_vector(NumElements: 8, ScalarTy: i16);
91 const LLT v4i16 = LLT::fixed_vector(NumElements: 4, ScalarTy: i16);
92
93 const LLT i32 = LLT::integer(SizeInBits: 32);
94 const LLT v2i32 = LLT::fixed_vector(NumElements: 2, ScalarTy: i32);
95 const LLT v4i32 = LLT::fixed_vector(NumElements: 4, ScalarTy: i32);
96
97 const LLT i64 = LLT::integer(SizeInBits: 64);
98 const LLT v2i64 = LLT::fixed_vector(NumElements: 2, ScalarTy: i64);
99
100 const LLT i128 = LLT::integer(SizeInBits: 128);
101
102 const LLT nxv16i8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: i8);
103 const LLT nxv8i16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: i16);
104 const LLT nxv4i32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: i32);
105 const LLT nxv2i64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: i64);
106
107 std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
108 v16s8, v8s16, v4s32,
109 v2s64, v2p0,
110 /* End 128bit types */
111 /* Begin 64bit types */
112 v8s8, v4s16, v2s32};
113 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
114 SmallVector<LLT, 8> PackedVectorAllTypesVec(PackedVectorAllTypeList);
115 SmallVector<LLT, 8> ScalarAndPtrTypesVec(ScalarAndPtrTypesList);
116
117 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
118
119 // FIXME: support subtargets which have neon/fp-armv8 disabled.
120 if (!ST.hasNEON() || !ST.hasFPARMv8())
121 return;
122
123 // Some instructions only support s16 if the subtarget has full 16-bit FP
124 // support.
125 const bool HasFP16 = ST.hasFullFP16();
126 const bool HasCSSC = ST.hasCSSC();
127 const bool HasRCPC3 = ST.hasRCPC3();
128 const bool HasSVE = ST.hasSVE();
129
130 getActionDefinitionsBuilder(
131 Opcodes: {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
132 .legalFor(Types: {p0, s8, s16, s32, s64, s128})
133 .legalFor(Types: {v2s8, v4s8, v8s8, v16s8, v2s16, v4s16, v8s16, v2s32, v4s32,
134 v2s64, v2p0})
135 .widenScalarToNextPow2(TypeIdx: 0)
136 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
137 .moreElementsToNextPow2(TypeIdx: 0)
138 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
139 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
140 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
141 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
142 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
143 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
144 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
145
146 getActionDefinitionsBuilder(Opcode: G_PHI)
147 .legalFor(Types: {p0, s16, s32, s64})
148 .legalFor(Types: PackedVectorAllTypeList)
149 .widenScalarToNextPow2(TypeIdx: 0)
150 .moreElementsToNextPow2(TypeIdx: 0)
151 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
152 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: s64)
153 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
154 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
155 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
156 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
157 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
158 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8);
159
160 getActionDefinitionsBuilder(Opcode: G_INSERT)
161 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64, p0}), P1: typeInSet(TypeIdx: 1, TypesInit: {s8, s16, s32}),
162 args: smallerThan(TypeIdx0: 1, TypeIdx1: 0)))
163 .widenScalarToNextPow2(TypeIdx: 0)
164 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
165 .widenScalarToNextPow2(TypeIdx: 1)
166 .minScalar(TypeIdx: 1, Ty: s8)
167 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s32}), TypeIdx: 1, Ty: s16)
168 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s64, p0}), TypeIdx: 1, Ty: s32);
169
170 getActionDefinitionsBuilder(Opcode: G_EXTRACT)
171 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64, p0}),
172 P1: typeInSet(TypeIdx: 1, TypesInit: {s32, s64, s128, p0}), args: smallerThan(TypeIdx0: 0, TypeIdx1: 1)))
173 .widenScalarToNextPow2(TypeIdx: 1)
174 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s128)
175 .widenScalarToNextPow2(TypeIdx: 0)
176 .minScalar(TypeIdx: 0, Ty: s16)
177 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s32}), TypeIdx: 0, Ty: s16)
178 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s64, p0}), TypeIdx: 0, Ty: s32)
179 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s128}), TypeIdx: 0, Ty: s64);
180
181 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB, G_AND, G_OR, G_XOR})
182 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
183 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
184 .widenScalarToNextPow2(TypeIdx: 0)
185 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
186 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
187 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
188 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
189 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
190 .minScalarOrEltIf(
191 Predicate: [=](const LegalityQuery &Query) {
192 return Query.Types[0].getNumElements() <= 2;
193 },
194 TypeIdx: 0, Ty: s32)
195 .minScalarOrEltIf(
196 Predicate: [=](const LegalityQuery &Query) {
197 return Query.Types[0].getNumElements() <= 4;
198 },
199 TypeIdx: 0, Ty: s16)
200 .minScalarOrEltIf(
201 Predicate: [=](const LegalityQuery &Query) {
202 return Query.Types[0].getNumElements() <= 16;
203 },
204 TypeIdx: 0, Ty: s8)
205 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
206 .moreElementsToNextPow2(TypeIdx: 0);
207
208 getActionDefinitionsBuilder(Opcode: G_MUL)
209 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
210 .widenScalarToNextPow2(TypeIdx: 0)
211 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
212 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
213 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
214 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
215 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
216 .minScalarOrEltIf(
217 Predicate: [=](const LegalityQuery &Query) {
218 return Query.Types[0].getNumElements() <= 2;
219 },
220 TypeIdx: 0, Ty: s32)
221 .minScalarOrEltIf(
222 Predicate: [=](const LegalityQuery &Query) {
223 return Query.Types[0].getNumElements() <= 4;
224 },
225 TypeIdx: 0, Ty: s16)
226 .minScalarOrEltIf(
227 Predicate: [=](const LegalityQuery &Query) {
228 return Query.Types[0].getNumElements() <= 16;
229 },
230 TypeIdx: 0, Ty: s8)
231 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
232 .moreElementsToNextPow2(TypeIdx: 0);
233
234 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
235 .customIf(Predicate: [=](const LegalityQuery &Query) {
236 const auto &SrcTy = Query.Types[0];
237 const auto &AmtTy = Query.Types[1];
238 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
239 AmtTy.getSizeInBits() == 32;
240 })
241 .legalFor(Types: {
242 {i32, i32},
243 {i32, i64},
244 {i64, i64},
245 {v8i8, v8i8},
246 {v16i8, v16i8},
247 {v4i16, v4i16},
248 {v8i16, v8i16},
249 {v2i32, v2i32},
250 {v4i32, v4i32},
251 {v2i64, v2i64},
252 })
253 .widenScalarToNextPow2(TypeIdx: 1)
254 .widenScalarToNextPow2(TypeIdx: 0)
255 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
256 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
257 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
258 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
259 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
260 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
261 .moreElementsToNextPow2(TypeIdx: 0)
262 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0)
263 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
264 .minScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, LargeTypeIdx: 0)
265 .maxScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, SmallTypeIdx: 0);
266
267 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
268 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
269 .clampScalarOrElt(TypeIdx: 1, MinTy: s64, MaxTy: s64)
270 .clampNumElements(TypeIdx: 0, MinTy: v2p0, MaxTy: v2p0);
271
272 getActionDefinitionsBuilder(Opcode: G_PTRMASK).legalFor(Types: {{p0, s64}});
273
274 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV})
275 .legalFor(Types: {i32, i64})
276 .libcallFor(Types: {i128})
277 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
278 .widenScalarToNextPow2(TypeIdx: 0)
279 .scalarize(TypeIdx: 0);
280
281 getActionDefinitionsBuilder(Opcodes: {G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
282 .lowerFor(Types: {i8, i16, i32, i64, v2i32, v4i32, v2i64})
283 .libcallFor(Types: {i128})
284 .widenScalarOrEltToNextPow2(TypeIdx: 0)
285 .minScalarOrElt(TypeIdx: 0, Ty: s32)
286 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
287 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
288 .scalarize(TypeIdx: 0);
289
290 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
291 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
292 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
293 .lower();
294
295 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
296 .legalFor(Types: {i64, v16i8, v8i16, v4i32})
297 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
298 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
299 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
300 .lower();
301
302 getActionDefinitionsBuilder(
303 Opcodes: {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT})
304 .lower();
305
306 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX})
307 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
308 .legalFor(Pred: HasCSSC, Types: {i32, i64})
309 .minScalar(Pred: HasCSSC, TypeIdx: 0, Ty: s32)
310 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
311 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
312 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
313 .lower();
314
315 // FIXME: Legal vector types are only legal with NEON.
316 getActionDefinitionsBuilder(Opcode: G_ABS)
317 .legalFor(Pred: HasCSSC, Types: {i32, i64})
318 .legalFor(Types: {v16i8, v8i16, v4i32, v2i64, v2p0, v8i8, v4i16, v2i32})
319 .customIf(Predicate: [=](const LegalityQuery &Q) {
320 // TODO: Fix suboptimal codegen for 128+ bit types.
321 LLT SrcTy = Q.Types[0];
322 return SrcTy.isScalar() && SrcTy.getSizeInBits() < 128;
323 })
324 .widenScalarIf(
325 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v4s8; },
326 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v4i16); })
327 .widenScalarIf(
328 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v2s16; },
329 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v2i32); })
330 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
331 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
332 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
333 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
334 .moreElementsToNextPow2(TypeIdx: 0)
335 .lower();
336
337 getActionDefinitionsBuilder(
338 Opcodes: {G_ABDS, G_ABDU, G_UAVGFLOOR, G_UAVGCEIL, G_SAVGFLOOR, G_SAVGCEIL})
339 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
340 .lower();
341
342 getActionDefinitionsBuilder(
343 Opcodes: {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
344 .legalFor(Types: {{i32, i32}, {i64, i32}})
345 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
346 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
347 .widenScalarToNextPow2(TypeIdx: 0)
348 .lower();
349
350 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR})
351 .customFor(Types: {{i32, i32}, {i32, i64}, {i64, i64}})
352 .lower();
353
354 getActionDefinitionsBuilder(Opcode: G_ROTR)
355 .legalFor(Types: {{i32, i64}, {i64, i64}})
356 .customIf(Predicate: [=](const LegalityQuery &Q) {
357 return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
358 })
359 .lower();
360 getActionDefinitionsBuilder(Opcode: G_ROTL).lower();
361
362 getActionDefinitionsBuilder(Opcodes: {G_SBFX, G_UBFX})
363 .customFor(Types: {{s32, s32}, {s64, s64}});
364
365 auto always = [=](const LegalityQuery &Q) { return true; };
366 getActionDefinitionsBuilder(Opcode: G_CTPOP)
367 .legalFor(Pred: HasCSSC, Types: {{i32, i32}, {i64, i64}})
368 .legalFor(Types: {{v8i8, v8i8}, {v16i8, v16i8}})
369 .customFor(Pred: !HasCSSC, Types: {{s32, s32}, {s64, s64}})
370 .customFor(Types: {{s128, s128},
371 {v4s16, v4s16},
372 {v8s16, v8s16},
373 {v2s32, v2s32},
374 {v4s32, v4s32},
375 {v2s64, v2s64}})
376 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s128)
377 .widenScalarToNextPow2(TypeIdx: 0)
378 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
379 .minScalarEltSameAsIf(Predicate: always, TypeIdx: 1, LargeTypeIdx: 0)
380 .maxScalarEltSameAsIf(Predicate: always, TypeIdx: 1, SmallTypeIdx: 0)
381 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
382 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
383 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
384 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
385 .moreElementsToNextPow2(TypeIdx: 0)
386 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
387
388 getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTLS})
389 .legalFor(Types: {{i32, i32},
390 {i64, i64},
391 {v8i8, v8i8},
392 {v16i8, v16i8},
393 {v4i16, v4i16},
394 {v8i16, v8i16},
395 {v2i32, v2i32},
396 {v4i32, v4i32}})
397 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
398 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
399 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, /*Min=*/MinSize: 8)
400 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
401 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
402 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
403 .moreElementsToNextPow2(TypeIdx: 0)
404 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 32), TypeIdx: 0)
405 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
406
407 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR).lower();
408
409 getActionDefinitionsBuilder(Opcode: G_CTLZ_ZERO_POISON).lower();
410
411 getActionDefinitionsBuilder(Opcode: G_CTTZ)
412 .lowerIf(Predicate: isVector(TypeIdx: 0))
413 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
414 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
415 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1)
416 .legalFor(Pred: HasCSSC, Types: {s32, s64})
417 .customFor(Pred: !HasCSSC, Types: {s32, s64});
418
419 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON).lower();
420
421 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
422 .legalFor(Types: {i32, i64, v8i8, v16i8})
423 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
424 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
425 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
426 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
427 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
428 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
429 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
430 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
431 .moreElementsToNextPow2(TypeIdx: 0)
432 .lower();
433
434 getActionDefinitionsBuilder(Opcode: G_CLMUL).legalFor(Types: {v8i8, v16i8});
435
436 getActionDefinitionsBuilder(Opcode: G_BSWAP)
437 .legalFor(Types: {i32, i64, v4i16, v8i16, v2i32, v4i32, v2i64})
438 .widenScalarOrEltToNextPow2(TypeIdx: 0, MinSize: 16)
439 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
440 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
441 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
442 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
443 .moreElementsToNextPow2(TypeIdx: 0);
444
445 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT})
446 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
447 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
448 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
449 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
450 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
451 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
452 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
453 .moreElementsToNextPow2(TypeIdx: 0)
454 .lower();
455
456 getActionDefinitionsBuilder(
457 Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
458 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
459 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
460 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
461 .legalFor(Pred: HasFP16, Types: {f16, v4f16, v8f16})
462 .libcallFor(Types: {f128})
463 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
464 .widenScalarIf(
465 Predicate: [=](const LegalityQuery &Q) {
466 return (!HasFP16 && Q.Types[0].getScalarType().isFloat16()) ||
467 Q.Types[0].getScalarType().isBFloat16();
468 },
469 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
470 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
471 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
472 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
473 .moreElementsToNextPow2(TypeIdx: 0);
474
475 getActionDefinitionsBuilder(Opcodes: {G_FABS, G_FNEG})
476 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
477 .legalFor(Pred: HasFP16, Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16})
478 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
479 .lowerIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64))
480 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
481 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
482 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
483 .moreElementsToNextPow2(TypeIdx: 0)
484 .lowerFor(Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16});
485
486 getActionDefinitionsBuilder(Opcodes: {G_FREM, G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
487 G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10,
488 G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH,
489 G_FSINH, G_FTANH, G_FMODF})
490 .libcallFor(Types: {f32, f64, f128})
491 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
492 .scalarize(TypeIdx: 0);
493 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
494 .libcallFor(Types: {{f32, i32}, {f64, i32}, {f128, i32}})
495 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
496 .scalarize(TypeIdx: 0);
497
498 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_INTRINSIC_LRINT})
499 .legalFor(Types: {{i32, f32}, {i32, f64}, {i64, f32}, {i64, f64}})
500 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {i64, f16}})
501 .minScalar(TypeIdx: 1, Ty: s32)
502 .libcallFor(Types: {{s64, s128}})
503 .lower();
504 getActionDefinitionsBuilder(Opcodes: {G_LLROUND, G_INTRINSIC_LLRINT})
505 .legalFor(Types: {{i64, f32}, {i64, f64}})
506 .legalFor(Pred: HasFP16, Types: {{i64, f16}})
507 .minScalar(TypeIdx: 0, Ty: s64)
508 .minScalar(TypeIdx: 1, Ty: s32)
509 .libcallFor(Types: {{s64, s128}})
510 .lower();
511
512 // TODO: Custom legalization for mismatched types.
513 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
514 .moreElementsIf(
515 Predicate: [](const LegalityQuery &Query) { return Query.Types[0].isScalar(); },
516 Mutation: [=](const LegalityQuery &Query) {
517 const LLT Ty = Query.Types[0];
518 return std::pair(0, LLT::fixed_vector(NumElements: Ty == s16 ? 4 : 2, ScalarTy: Ty));
519 })
520 .lower();
521
522 getActionDefinitionsBuilder(Opcode: G_FMAD).lower();
523
524 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
525 auto &Actions = getActionDefinitionsBuilder(Opcode: Op);
526
527 if (Op == G_SEXTLOAD)
528 Actions.lowerIf(Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered));
529
530 // Atomics have zero extending behavior.
531 Actions
532 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8},
533 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8},
534 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
535 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 2},
536 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 2},
537 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 4},
538 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
539 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
540 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}})
541 .widenScalarToNextPow2(TypeIdx: 0)
542 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
543 // TODO: We could support sum-of-pow2's but the lowering code doesn't know
544 // how to do that yet.
545 .unsupportedIfMemSizeNotPow2()
546 // Lower anything left over into G_*EXT and G_LOAD
547 .lower();
548 }
549
550 auto IsPtrVecPred = [=](const LegalityQuery &Query) {
551 const LLT &ValTy = Query.Types[0];
552 return ValTy.isPointerVector() && ValTy.getAddressSpace() == 0;
553 };
554
555 getActionDefinitionsBuilder(Opcode: G_LOAD)
556 .customIf(Predicate: [=](const LegalityQuery &Query) {
557 return HasRCPC3 && Query.Types[0] == s128 &&
558 Query.MMODescrs[0].Ordering == AtomicOrdering::Acquire;
559 })
560 .customIf(Predicate: [=](const LegalityQuery &Query) {
561 return Query.Types[0] == s128 &&
562 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
563 })
564 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8},
565 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8},
566 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
567 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
568 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
569 {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8},
570 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8},
571 {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
572 {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8},
573 {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
574 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8},
575 {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8},
576 {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
577 // These extends are also legal
578 .legalForTypesWithMemDesc(
579 TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}})
580 .legalForTypesWithMemDesc(TypesAndMemDesc: {
581 // SVE vscale x 128 bit base sizes
582 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
583 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
584 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
585 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
586 })
587 .widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
588 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
589 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
590 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
591 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
592 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
593 .lowerIfMemSizeNotByteSizePow2()
594 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
595 .narrowScalarIf(
596 Predicate: [=](const LegalityQuery &Query) {
597 // Clamp extending load results to 32-bits.
598 return Query.Types[0].isScalar() &&
599 Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
600 Query.Types[0].getSizeInBits() > 32;
601 },
602 Mutation: changeTo(TypeIdx: 0, Ty: i32))
603 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
604 .bitcastIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
605 Mutation: [=](const LegalityQuery &Query) {
606 const LLT VecTy = Query.Types[0];
607 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
608 })
609 .customIf(Predicate: IsPtrVecPred)
610 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
611 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
612
613 getActionDefinitionsBuilder(Opcode: G_STORE)
614 .customIf(Predicate: [=](const LegalityQuery &Query) {
615 return HasRCPC3 && Query.Types[0] == s128 &&
616 Query.MMODescrs[0].Ordering == AtomicOrdering::Release;
617 })
618 .customIf(Predicate: [=](const LegalityQuery &Query) {
619 return Query.Types[0] == s128 &&
620 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
621 })
622 .widenScalarIf(
623 Predicate: all(P0: scalarNarrowerThan(TypeIdx: 0, Size: 32),
624 P1: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Release)),
625 Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: s32))
626 .legalForTypesWithMemDesc(
627 TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s16
628 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s32
629 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s64
630 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s32
631 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s64
632 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
633 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}, // truncstorei32 from s64
634 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
635 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
636 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
637 .legalForTypesWithMemDesc(TypesAndMemDesc: {
638 // SVE vscale x 128 bit base sizes
639 // TODO: Add nxv2p0. Consider bitcastIf.
640 // See #92130
641 // https://github.com/llvm/llvm-project/pull/92130#discussion_r1616888461
642 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
643 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
644 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
645 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
646 })
647 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
648 .minScalarOrElt(TypeIdx: 0, Ty: s8)
649 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
650 return Query.Types[0].isScalar() &&
651 Query.Types[0] != Query.MMODescrs[0].MemoryTy;
652 })
653 // Maximum: sN * k = 128
654 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
655 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
656 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
657 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
658 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
659 .lowerIfMemSizeNotPow2()
660 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
661 .bitcastIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
662 P1: LegalityPredicate([=](const LegalityQuery &Query) {
663 return Query.Types[0].getSizeInBits() ==
664 Query.MMODescrs[0].MemoryTy.getSizeInBits();
665 })),
666 Mutation: [=](const LegalityQuery &Query) {
667 const LLT VecTy = Query.Types[0];
668 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
669 })
670 .customIf(Predicate: IsPtrVecPred)
671 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
672 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
673 .lower();
674
675 getActionDefinitionsBuilder(Opcode: G_INDEXED_STORE)
676 // Idx 0 == Ptr, Idx 1 == Val
677 // TODO: we can implement legalizations but as of now these are
678 // generated in a very specific way.
679 .legalForTypesWithMemDesc(TypesAndMemDesc: {
680 {.Type0: p0, .Type1: s8, .MemTy: s8, .Align: 8},
681 {.Type0: p0, .Type1: s16, .MemTy: s16, .Align: 8},
682 {.Type0: p0, .Type1: s32, .MemTy: s8, .Align: 8},
683 {.Type0: p0, .Type1: s32, .MemTy: s16, .Align: 8},
684 {.Type0: p0, .Type1: s32, .MemTy: s32, .Align: 8},
685 {.Type0: p0, .Type1: s64, .MemTy: s64, .Align: 8},
686 {.Type0: p0, .Type1: p0, .MemTy: p0, .Align: 8},
687 {.Type0: p0, .Type1: v8s8, .MemTy: v8s8, .Align: 8},
688 {.Type0: p0, .Type1: v16s8, .MemTy: v16s8, .Align: 8},
689 {.Type0: p0, .Type1: v4s16, .MemTy: v4s16, .Align: 8},
690 {.Type0: p0, .Type1: v8s16, .MemTy: v8s16, .Align: 8},
691 {.Type0: p0, .Type1: v2s32, .MemTy: v2s32, .Align: 8},
692 {.Type0: p0, .Type1: v4s32, .MemTy: v4s32, .Align: 8},
693 {.Type0: p0, .Type1: v2s64, .MemTy: v2s64, .Align: 8},
694 {.Type0: p0, .Type1: v2p0, .MemTy: v2p0, .Align: 8},
695 {.Type0: p0, .Type1: s128, .MemTy: s128, .Align: 8},
696 })
697 .unsupported();
698
699 auto IndexedLoadBasicPred = [=](const LegalityQuery &Query) {
700 LLT LdTy = Query.Types[0];
701 LLT PtrTy = Query.Types[1];
702 if (!llvm::is_contained(Range: PackedVectorAllTypesVec, Element: LdTy) &&
703 !llvm::is_contained(Range: ScalarAndPtrTypesVec, Element: LdTy) && LdTy != s128)
704 return false;
705 if (PtrTy != p0)
706 return false;
707 return true;
708 };
709 getActionDefinitionsBuilder(Opcode: G_INDEXED_LOAD)
710 .unsupportedIf(
711 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
712 .legalIf(Predicate: IndexedLoadBasicPred)
713 .unsupported();
714 getActionDefinitionsBuilder(Opcodes: {G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD})
715 .unsupportedIf(
716 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
717 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64}),
718 P1: LegalityPredicate([=](const LegalityQuery &Q) {
719 LLT LdTy = Q.Types[0];
720 LLT PtrTy = Q.Types[1];
721 LLT MemTy = Q.MMODescrs[0].MemoryTy;
722 if (PtrTy != p0)
723 return false;
724 if (LdTy == s16)
725 return MemTy == s8;
726 if (LdTy == s32)
727 return MemTy == s8 || MemTy == s16;
728 if (LdTy == s64)
729 return MemTy == s8 || MemTy == s16 || MemTy == s32;
730 return false;
731 })))
732 .unsupported();
733
734 // Constants
735 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
736 .legalFor(Types: {p0, s8, s16, s32, s64})
737 .widenScalarToNextPow2(TypeIdx: 0)
738 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64);
739 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
740 .legalFor(Types: {s16, s32, s64, s128});
741
742 // FIXME: fix moreElementsToNextPow2
743 getActionDefinitionsBuilder(Opcode: G_ICMP)
744 .legalFor(Types: {{i32, i32}, {i32, i64}, {i32, p0}})
745 .widenScalarOrEltToNextPow2(TypeIdx: 1)
746 .minScalarOrElt(TypeIdx: 1, Ty: s8)
747 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
748 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
749 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
750 .minScalarEltSameAsIf(
751 Predicate: [=](const LegalityQuery &Query) {
752 const LLT &Ty = Query.Types[0];
753 const LLT &SrcTy = Query.Types[1];
754 return Ty.isVector() && !SrcTy.isPointerVector() &&
755 Ty.getElementType() != SrcTy.getElementType();
756 },
757 TypeIdx: 0, LargeTypeIdx: 1)
758 .minScalarOrEltIf(
759 Predicate: [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
760 TypeIdx: 1, Ty: s32)
761 .minScalarOrEltIf(
762 Predicate: [=](const LegalityQuery &Query) {
763 return Query.Types[1].isPointerVector();
764 },
765 TypeIdx: 0, Ty: s64)
766 .moreElementsToNextPow2(TypeIdx: 1)
767 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
768 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
769 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
770 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
771 .clampNumElements(TypeIdx: 1, MinTy: v2p0, MaxTy: v2p0)
772 .customIf(Predicate: isVector(TypeIdx: 0));
773
774 getActionDefinitionsBuilder(Opcode: G_FCMP)
775 .legalFor(Types: {{i32, f32},
776 {i32, f64},
777 {v4i32, v4f32},
778 {v2i32, v2f32},
779 {v2i64, v2f64}})
780 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
781 .widenScalarOrEltToNextPow2(TypeIdx: 1)
782 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
783 .widenScalarIf(
784 Predicate: [=](const LegalityQuery &Q) {
785 return (!HasFP16 && Q.Types[1].getScalarType().isFloat16()) ||
786 Q.Types[1].getScalarType().isBFloat16();
787 },
788 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
789 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
790 .minScalarEltSameAsIf(
791 Predicate: [=](const LegalityQuery &Query) {
792 const LLT &Ty = Query.Types[0];
793 const LLT &SrcTy = Query.Types[1];
794 return Ty.isVector() && !SrcTy.isPointerVector() &&
795 Ty.getElementType() != SrcTy.getElementType();
796 },
797 TypeIdx: 0, LargeTypeIdx: 1)
798 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
799 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
800 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
801 .moreElementsToNextPow2(TypeIdx: 1)
802 .libcallFor(Types: {{s32, s128}});
803
804 // Extensions
805 auto ExtLegalFunc = [=](const LegalityQuery &Query) {
806 unsigned DstSize = Query.Types[0].getSizeInBits();
807
808 // Handle legal vectors using legalFor
809 if (Query.Types[0].isVector())
810 return false;
811
812 if (DstSize < 8 || DstSize >= 128 || !isPowerOf2_32(Value: DstSize))
813 return false; // Extending to a scalar s128 needs narrowing.
814
815 const LLT &SrcTy = Query.Types[1];
816
817 // Make sure we fit in a register otherwise. Don't bother checking that
818 // the source type is below 128 bits. We shouldn't be allowing anything
819 // through which is wider than the destination in the first place.
820 unsigned SrcSize = SrcTy.getSizeInBits();
821 if (SrcSize < 8 || !isPowerOf2_32(Value: SrcSize))
822 return false;
823
824 return true;
825 };
826 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
827 .legalIf(Predicate: ExtLegalFunc)
828 .legalFor(Types: {{v8s16, v8s8}, {v4s32, v4s16}, {v2s64, v2s32}})
829 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64) // Just for s128, others are handled above.
830 .moreElementsToNextPow2(TypeIdx: 0)
831 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
832 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
833 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
834 // Tries to convert a large EXTEND into two smaller EXTENDs
835 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
836 return (Query.Types[0].getScalarSizeInBits() >
837 Query.Types[1].getScalarSizeInBits() * 2) &&
838 Query.Types[0].isVector() &&
839 (Query.Types[1].getScalarSizeInBits() == 8 ||
840 Query.Types[1].getScalarSizeInBits() == 16);
841 })
842 .clampMinNumElements(TypeIdx: 1, EltTy: s8, MinElements: 8)
843 .clampMinNumElements(TypeIdx: 1, EltTy: s16, MinElements: 4)
844 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
845
846 getActionDefinitionsBuilder(Opcode: G_TRUNC)
847 .legalFor(Types: {{v8s8, v8s16}, {v4s16, v4s32}, {v2s32, v2s64}})
848 .moreElementsToNextPow2(TypeIdx: 0)
849 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 8)
850 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 4)
851 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 2)
852 .minScalarOrEltIf(
853 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
854 TypeIdx: 0, Ty: s8)
855 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
856 LLT DstTy = Query.Types[0];
857 LLT SrcTy = Query.Types[1];
858 return DstTy.isVector() && SrcTy.getSizeInBits() > 128 &&
859 DstTy.getScalarSizeInBits() * 2 <= SrcTy.getScalarSizeInBits();
860 })
861 .clampMinNumElements(TypeIdx: 0, EltTy: s8, MinElements: 8)
862 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 4)
863 .alwaysLegal();
864
865 getActionDefinitionsBuilder(Opcodes: {G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
866 .legalFor(Types: {{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
867 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v8s8)
868 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v4s16)
869 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v2s32)
870 .lower();
871
872 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
873 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
874 .maxScalar(TypeIdx: 0, Ty: s64)
875 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
876 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
877 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
878 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
879 .lower();
880
881 // FP conversions
882 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
883 .legalFor(
884 Types: {{f16, f32}, {f16, f64}, {f32, f64}, {v4f16, v4f32}, {v2f32, v2f64}})
885 .legalFor(Pred: ST.hasBF16(), Types: {{bf16, f32}, {v4bf16, v4f32}})
886 .libcallFor(Types: {{f16, f128}, {f32, f128}, {f64, f128}})
887 .moreElementsToNextPow2(TypeIdx: 1)
888 .customIf(Predicate: [](const LegalityQuery &Q) {
889 LLT DstTy = Q.Types[0];
890 LLT SrcTy = Q.Types[1];
891 return SrcTy.getScalarSizeInBits() == 64 &&
892 DstTy.getScalarSizeInBits() == 16;
893 })
894 .lowerFor(Types: {{bf16, f32}, {v4bf16, v4f32}})
895 // Clamp based on input
896 .clampNumElements(TypeIdx: 1, MinTy: v4s32, MaxTy: v4s32)
897 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
898 .scalarize(TypeIdx: 0);
899
900 getActionDefinitionsBuilder(Opcode: G_FPEXT)
901 .legalFor(Types: {{f32, f16},
902 {f64, f16},
903 {f32, bf16},
904 {f64, f32},
905 {v4f32, v4f16},
906 {v4f32, v4bf16},
907 {v2f64, v2f32}})
908 .libcallFor(Types: {{f128, f64}, {f128, f32}, {f128, f16}})
909 .moreElementsToNextPow2(TypeIdx: 0)
910 .widenScalarIf(
911 Predicate: [](const LegalityQuery &Q) {
912 LLT DstTy = Q.Types[0];
913 LLT SrcTy = Q.Types[1];
914 return SrcTy.isVector() && DstTy.isVector() &&
915 SrcTy.getScalarSizeInBits() == 16 &&
916 DstTy.getScalarSizeInBits() == 64;
917 },
918 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
919 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
920 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
921 .scalarize(TypeIdx: 0);
922
923 // Conversions
924 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
925 .legalFor(Types: {{i32, f32},
926 {i64, f32},
927 {i32, f64},
928 {i64, f64},
929 {v2i32, v2f32},
930 {v4i32, v4f32},
931 {v2i64, v2f64}})
932 .legalFor(Pred: HasFP16,
933 Types: {{i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
934 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
935 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
936 // The range of a fp16 value fits into an i17, so we can lower the width
937 // to i64.
938 .narrowScalarIf(
939 Predicate: [=](const LegalityQuery &Query) {
940 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
941 },
942 Mutation: changeTo(TypeIdx: 0, Ty: i64))
943 .moreElementsToNextPow2(TypeIdx: 0)
944 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0)
945 .minScalar(TypeIdx: 0, Ty: s32)
946 .widenScalarIf(
947 Predicate: [HasFP16](const LegalityQuery &Query) {
948 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
949 Query.Types[1].getScalarType().isBFloat16();
950 },
951 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
952 .widenScalarIf(
953 Predicate: [=](const LegalityQuery &Query) {
954 return Query.Types[0].getScalarSizeInBits() <= 64 &&
955 Query.Types[0].getScalarSizeInBits() >
956 Query.Types[1].getScalarSizeInBits();
957 },
958 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
959 .widenScalarIf(
960 Predicate: [=](const LegalityQuery &Query) {
961 return Query.Types[1].getScalarSizeInBits() <= 64 &&
962 Query.Types[0].getScalarSizeInBits() <
963 Query.Types[1].getScalarSizeInBits();
964 },
965 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
966 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
967 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
968 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
969 .libcallFor(
970 Types: {{i32, f128}, {i64, f128}, {i128, f128}, {i128, f32}, {i128, f64}});
971
972 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
973 .legalFor(Types: {{i32, f32},
974 {i64, f32},
975 {i32, f64},
976 {i64, f64},
977 {v2i32, v2f32},
978 {v4i32, v4f32},
979 {v2i64, v2f64}})
980 .legalFor(
981 Pred: HasFP16,
982 Types: {{i16, f16}, {i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
983 // Handle types larger than i64 by scalarizing/lowering.
984 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
985 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
986 // NOTE: narrowScalarIf is invalid for the saturating conversions,
987 // infinities need the full width of the target integer type.
988 .lowerIf(Predicate: ::any(P0: scalarWiderThan(TypeIdx: 0, Size: 64), P1: scalarWiderThan(TypeIdx: 1, Size: 64)), Mutation: 0)
989 .moreElementsToNextPow2(TypeIdx: 0)
990 .widenScalarToNextPow2(TypeIdx: 0, /*MinSize=*/32)
991 .minScalar(TypeIdx: 0, Ty: s32)
992 .widenScalarIf(
993 Predicate: [HasFP16](const LegalityQuery &Query) {
994 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
995 Query.Types[1].getScalarType().isBFloat16();
996 },
997 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
998 .widenScalarIf(
999 Predicate: [=](const LegalityQuery &Query) {
1000 unsigned ITySize = Query.Types[0].getScalarSizeInBits();
1001 return (ITySize == 16 || ITySize == 32 || ITySize == 64) &&
1002 ITySize > Query.Types[1].getScalarSizeInBits();
1003 },
1004 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1005 .widenScalarIf(
1006 Predicate: [=](const LegalityQuery &Query) {
1007 unsigned FTySize = Query.Types[1].getScalarSizeInBits();
1008 return (FTySize == 16 || FTySize == 32 || FTySize == 64) &&
1009 Query.Types[0].getScalarSizeInBits() < FTySize;
1010 },
1011 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1012 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1013 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1014 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1015 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1016
1017 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
1018 .legalFor(Types: {{f32, i32},
1019 {f64, i32},
1020 {f32, i64},
1021 {f64, i64},
1022 {v2f32, v2i32},
1023 {v4f32, v4i32},
1024 {v2f64, v2i64}})
1025 .legalFor(Pred: HasFP16,
1026 Types: {{f16, i32}, {f16, i64}, {v4f16, v4i16}, {v8f16, v8i16}})
1027 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1028 return Query.Types[0].getScalarType().isBFloat16();
1029 })
1030 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
1031 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1032 .moreElementsToNextPow2(TypeIdx: 1)
1033 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1)
1034 .minScalar(TypeIdx: 1, Ty: f32)
1035 .lowerIf(Predicate: [](const LegalityQuery &Query) {
1036 return Query.Types[1].isVector() &&
1037 Query.Types[1].getScalarSizeInBits() == 64 &&
1038 Query.Types[0].getScalarSizeInBits() == 16;
1039 })
1040 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, /*MinSize=*/HasFP16 ? 16 : 32)
1041 .scalarizeIf(
1042 // v2i64->v2f32 needs to scalarize to avoid double-rounding issues.
1043 Predicate: [](const LegalityQuery &Query) {
1044 return Query.Types[0].getScalarSizeInBits() == 32 &&
1045 Query.Types[1].getScalarSizeInBits() == 64;
1046 },
1047 TypeIdx: 0)
1048 .widenScalarIf(
1049 Predicate: [](const LegalityQuery &Query) {
1050 return Query.Types[1].getScalarSizeInBits() <= 64 &&
1051 Query.Types[0].getScalarSizeInBits() <
1052 Query.Types[1].getScalarSizeInBits();
1053 },
1054 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1055 .widenScalarIf(
1056 Predicate: [](const LegalityQuery &Query) {
1057 return Query.Types[0].getScalarSizeInBits() <= 64 &&
1058 Query.Types[0].getScalarSizeInBits() >
1059 Query.Types[1].getScalarSizeInBits();
1060 },
1061 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1062 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1063 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1064 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1065 .libcallFor(Types: {{f16, i128},
1066 {f32, i128},
1067 {f64, i128},
1068 {f128, i128},
1069 {f128, i32},
1070 {f128, i64}});
1071
1072 // Control-flow
1073 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
1074 getActionDefinitionsBuilder(Opcode: G_BRCOND)
1075 .legalFor(Types: {s32})
1076 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32);
1077 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
1078
1079 getActionDefinitionsBuilder(Opcode: G_SELECT)
1080 .legalFor(Types: {{s32, s32}, {s64, s32}, {p0, s32}})
1081 .widenScalarToNextPow2(TypeIdx: 0)
1082 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
1083 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s32)
1084 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1085 .minScalarEltSameAsIf(Predicate: all(P0: isVector(TypeIdx: 0), P1: isVector(TypeIdx: 1)), TypeIdx: 1, LargeTypeIdx: 0)
1086 .lowerIf(Predicate: isVector(TypeIdx: 0));
1087
1088 // Pointer-handling
1089 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
1090
1091 if (TM.getCodeModel() == CodeModel::Small)
1092 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).custom();
1093 else
1094 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: {p0});
1095
1096 getActionDefinitionsBuilder(Opcode: G_PTRAUTH_GLOBAL_VALUE)
1097 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: p0), P1: typeIs(TypeIdx: 1, TypesInit: p0)));
1098
1099 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
1100 .legalFor(Types: {{i64, p0}, {v2i64, v2p0}})
1101 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 64)
1102 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64)
1103 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1104
1105 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
1106 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1107 return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
1108 })
1109 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
1110 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2);
1111
1112 // Casts for 32 and 64-bit width type are just copies.
1113 // Same for 128-bit width type, except they are on the FPR bank.
1114 getActionDefinitionsBuilder(Opcode: G_BITCAST)
1115 .legalForCartesianProduct(Types: {s16})
1116 // Keeping 32-bit instructions legal to prevent regression in some tests
1117 .legalForCartesianProduct(Types: {s32, v2s16, v4s8})
1118 .legalForCartesianProduct(Types: {s64, v8s8, v4s16, v2s32})
1119 .legalForCartesianProduct(Types: {s128, v16s8, v8s16, v4s32, v2s64, v2p0})
1120 .customIf(Predicate: [=](const LegalityQuery &Query) {
1121 // Handle casts from i1 vectors to scalars.
1122 LLT DstTy = Query.Types[0];
1123 LLT SrcTy = Query.Types[1];
1124 return DstTy.isScalar() && SrcTy.isVector() &&
1125 SrcTy.getScalarSizeInBits() == 1;
1126 })
1127 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
1128 return Query.Types[0].isVector() != Query.Types[1].isVector();
1129 })
1130 // moreElementsToNextPow2 cannot pad the source to match, so lower
1131 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
1132 LLT DstTy = Query.Types[0];
1133 LLT SrcTy = Query.Types[1];
1134 if (!DstTy.isFixedVector() || !SrcTy.isFixedVector())
1135 return false;
1136 unsigned MoreElts = 1u << Log2_32_Ceil(Value: DstTy.getNumElements());
1137 return SrcTy.getNumElements() * MoreElts % DstTy.getNumElements() != 0;
1138 })
1139 .moreElementsToNextPow2(TypeIdx: 0)
1140 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1141 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1142 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1143 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1144 .lower();
1145
1146 getActionDefinitionsBuilder(Opcode: G_VASTART).legalFor(Types: {p0});
1147
1148 // va_list must be a pointer, but most sized types are pretty easy to handle
1149 // as the destination.
1150 getActionDefinitionsBuilder(Opcode: G_VAARG)
1151 .customForCartesianProduct(Types0: {s8, s16, s32, s64, p0}, Types1: {p0})
1152 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
1153 .widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8);
1154
1155 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
1156 .lowerIf(
1157 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64, s128}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
1158
1159 bool UseOutlineAtomics = ST.outlineAtomics() && !ST.hasLSE();
1160
1161 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG)
1162 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1163 .customFor(Pred: !UseOutlineAtomics, Types: {{s128, p0}})
1164 .libcallFor(Pred: UseOutlineAtomics,
1165 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {s128, p0}})
1166 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1167
1168 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD,
1169 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
1170 G_ATOMICRMW_XOR})
1171 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1172 .libcallFor(Pred: UseOutlineAtomics,
1173 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
1174 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1175
1176 // Do not outline these atomics operations, as per comment in
1177 // AArch64ISelLowering.cpp's shouldExpandAtomicRMWInIR().
1178 getActionDefinitionsBuilder(
1179 Opcodes: {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
1180 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64}), P1: typeIs(TypeIdx: 1, TypesInit: p0)))
1181 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1182
1183 getActionDefinitionsBuilder(Opcode: G_BLOCK_ADDR).legalFor(Types: {p0});
1184
1185 // Merge/Unmerge
1186 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
1187 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
1188 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
1189 getActionDefinitionsBuilder(Opcode: Op)
1190 .widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: 8)
1191 // Above s64 lowered shifts narrow back to a merge and never terminate
1192 .lowerIf(Predicate: [=](const LegalityQuery &Q) {
1193 const LLT BigTy = Q.Types[BigTyIdx];
1194 return BigTy.isScalar() && !isPowerOf2_32(Value: BigTy.getSizeInBits()) &&
1195 BigTy.getSizeInBits() < 64;
1196 })
1197 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: 32)
1198 .clampScalar(TypeIdx: LitTyIdx, MinTy: s8, MaxTy: s64)
1199 .clampScalar(TypeIdx: BigTyIdx, MinTy: s32, MaxTy: s128)
1200 .legalIf(Predicate: [=](const LegalityQuery &Q) {
1201 switch (Q.Types[BigTyIdx].getSizeInBits()) {
1202 case 32:
1203 case 64:
1204 case 128:
1205 break;
1206 default:
1207 return false;
1208 }
1209 switch (Q.Types[LitTyIdx].getSizeInBits()) {
1210 case 8:
1211 case 16:
1212 case 32:
1213 case 64:
1214 return true;
1215 default:
1216 return false;
1217 }
1218 });
1219 }
1220
1221 // TODO : nxv4s16, nxv2s16, nxv2s32
1222 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
1223 .legalFor(Pred: HasSVE, Types: {{s16, nxv16s8, s64},
1224 {s16, nxv8s16, s64},
1225 {s32, nxv4s32, s64},
1226 {s64, nxv2s64, s64}})
1227 .unsupportedIf(Predicate: [=](const LegalityQuery &Query) {
1228 const LLT &EltTy = Query.Types[1].getElementType();
1229 if (Query.Types[1].isScalableVector())
1230 return false;
1231 return Query.Types[0] != EltTy;
1232 })
1233 .minScalar(TypeIdx: 2, Ty: s64)
1234 .customIf(Predicate: [=](const LegalityQuery &Query) {
1235 const LLT &VecTy = Query.Types[1];
1236 return VecTy == v8s8 || VecTy == v16s8 || VecTy == v2s16 ||
1237 VecTy == v4s16 || VecTy == v8s16 || VecTy == v2s32 ||
1238 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2p0;
1239 })
1240 .minScalarOrEltIf(
1241 Predicate: [=](const LegalityQuery &Query) {
1242 // We want to promote to <M x s1> to <M x s64> if that wouldn't
1243 // cause the total vec size to be > 128b.
1244 return Query.Types[1].isFixedVector() &&
1245 Query.Types[1].getNumElements() <= 2;
1246 },
1247 TypeIdx: 0, Ty: s64)
1248 .minScalarOrEltIf(
1249 Predicate: [=](const LegalityQuery &Query) {
1250 return Query.Types[1].isFixedVector() &&
1251 Query.Types[1].getNumElements() <= 4;
1252 },
1253 TypeIdx: 0, Ty: s32)
1254 .minScalarOrEltIf(
1255 Predicate: [=](const LegalityQuery &Query) {
1256 return Query.Types[1].isFixedVector() &&
1257 Query.Types[1].getNumElements() <= 8;
1258 },
1259 TypeIdx: 0, Ty: s16)
1260 .minScalarOrEltIf(
1261 Predicate: [=](const LegalityQuery &Query) {
1262 return Query.Types[1].isFixedVector() &&
1263 Query.Types[1].getNumElements() <= 16;
1264 },
1265 TypeIdx: 0, Ty: s8)
1266 .minScalarOrElt(TypeIdx: 0, Ty: s8) // Worst case, we need at least s8.
1267 .moreElementsToNextPow2(TypeIdx: 1)
1268 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1269 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1270 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1271 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1272 .clampMaxNumElements(TypeIdx: 1, EltTy: p0, MaxElements: 2)
1273 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1);
1274
1275 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
1276 .legalIf(
1277 Predicate: typeInSet(TypeIdx: 0, TypesInit: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64, v2p0}))
1278 .legalFor(Pred: HasSVE, Types: {{nxv16s8, s32, s64},
1279 {nxv8s16, s32, s64},
1280 {nxv4s32, s32, s64},
1281 {nxv2s64, s64, s64}})
1282 .moreElementsToNextPow2(TypeIdx: 0)
1283 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1284 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1285 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1286 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1287 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1288 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
1289 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
1290
1291 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
1292 .legalFor(Types: {{v8s8, s8},
1293 {v16s8, s8},
1294 {v4s16, s16},
1295 {v8s16, s16},
1296 {v2s32, s32},
1297 {v4s32, s32},
1298 {v2s64, s64},
1299 {v2p0, p0}})
1300 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1301 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1302 .minScalarOrElt(TypeIdx: 0, Ty: s8)
1303 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1304 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1305 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0);
1306
1307 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR_TRUNC).lower();
1308
1309 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
1310 .legalFor(Types: {{v8s8, s8},
1311 {v8s8, s32},
1312 {v16s8, s8},
1313 {v16s8, s32},
1314 {v4s16, s16},
1315 {v4s16, s32},
1316 {v8s16, s16},
1317 {v8s16, s32},
1318 {v2s32, s32},
1319 {v4s32, s32},
1320 {v2s64, s64},
1321 {v2p0, p0}});
1322
1323 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
1324 .legalIf(Predicate: [=](const LegalityQuery &Query) {
1325 const LLT &DstTy = Query.Types[0];
1326 const LLT &SrcTy = Query.Types[1];
1327 // For now just support the TBL2 variant which needs the source vectors
1328 // to be the same size as the dest.
1329 if (DstTy != SrcTy)
1330 return false;
1331 return llvm::is_contained(
1332 Set: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64}, Element: DstTy);
1333 })
1334 .moreElementsIf(
1335 Predicate: [](const LegalityQuery &Query) {
1336 return Query.Types[0].getNumElements() >
1337 Query.Types[1].getNumElements();
1338 },
1339 Mutation: changeTo(TypeIdx: 1, FromTypeIdx: 0))
1340 .moreElementsToNextPow2(TypeIdx: 0)
1341 .moreElementsIf(
1342 Predicate: [](const LegalityQuery &Query) {
1343 return Query.Types[0].getNumElements() <
1344 Query.Types[1].getNumElements();
1345 },
1346 Mutation: changeTo(TypeIdx: 0, FromTypeIdx: 1))
1347 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
1348 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1349 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1350 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1351 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1352 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1353 .bitcastIf(Predicate: isPointerVector(TypeIdx: 0), Mutation: [=](const LegalityQuery &Query) {
1354 // Bitcast pointers vector to i64.
1355 const LLT DstTy = Query.Types[0];
1356 return std::pair(
1357 0, LLT::vector(EC: DstTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 64)));
1358 });
1359
1360 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
1361 .legalFor(Types: {{v16s8, v8s8}, {v8s16, v4s16}, {v4s32, v2s32}})
1362 .customIf(Predicate: [=](const LegalityQuery &Query) {
1363 return Query.Types[0].isFixedVector() &&
1364 Query.Types[0].getScalarSizeInBits() < 8;
1365 })
1366 .bitcastIf(
1367 Predicate: [=](const LegalityQuery &Query) {
1368 return Query.Types[0].isFixedVector() &&
1369 Query.Types[1].isFixedVector() &&
1370 Query.Types[0].getScalarSizeInBits() >= 8 &&
1371 isPowerOf2_64(Value: Query.Types[0].getScalarSizeInBits()) &&
1372 Query.Types[0].getSizeInBits() <= 128 &&
1373 Query.Types[1].getSizeInBits() <= 64;
1374 },
1375 Mutation: [=](const LegalityQuery &Query) {
1376 const LLT DstTy = Query.Types[0];
1377 const LLT SrcTy = Query.Types[1];
1378 return std::pair(
1379 0, DstTy.changeElementSize(NewEltSize: SrcTy.getSizeInBits())
1380 .changeElementCount(
1381 EC: DstTy.getElementCount().divideCoefficientBy(
1382 RHS: SrcTy.getNumElements())));
1383 });
1384
1385 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
1386 .legalFor(Types: {{v8s8, v16s8}, {v4s16, v8s16}, {v2s32, v4s32}})
1387 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1388 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
1389 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
1390 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
1391 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
1392 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
1393 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
1394 .lower()
1395 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1396
1397 // TODO: {nxv16s8, s8}, {nxv8s16, s16}
1398 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
1399 .legalFor(Pred: HasSVE, Types: {{nxv4s32, s32}, {nxv2s64, s64}});
1400
1401 getActionDefinitionsBuilder(Opcode: G_JUMP_TABLE).legalFor(Types: {p0});
1402
1403 getActionDefinitionsBuilder(Opcode: G_BRJT).legalFor(Types: {{p0, s64}});
1404
1405 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
1406
1407 getActionDefinitionsBuilder(Opcode: G_DYN_STACKALLOC).custom();
1408
1409 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).lower();
1410
1411 if (ST.hasMOPS()) {
1412 // G_BZERO is not supported. Currently it is only emitted by
1413 // PreLegalizerCombiner for G_MEMSET with zero constant.
1414 getActionDefinitionsBuilder(Opcode: G_BZERO).unsupported();
1415
1416 getActionDefinitionsBuilder(Opcode: G_MEMSET)
1417 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1418 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64})
1419 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1420
1421 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE})
1422 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64})
1423 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1424
1425 // G_MEMCPY_INLINE does not have a tailcall immediate
1426 getActionDefinitionsBuilder(Opcode: G_MEMCPY_INLINE)
1427 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64});
1428
1429 getActionDefinitionsBuilder(Opcode: G_MEMSET_INLINE)
1430 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1431 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64});
1432 } else {
1433 getActionDefinitionsBuilder(Opcodes: {G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
1434 .libcall();
1435 }
1436
1437 // For fadd reductions we have pairwise operations available. We treat the
1438 // usual legal types as legal and handle the lowering to pairwise instructions
1439 // later.
1440 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FADD)
1441 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1442 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1443 .widenScalarIf(
1444 Predicate: [HasFP16](const LegalityQuery &Query) {
1445 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1446 Query.Types[0].getScalarType().isBFloat16();
1447 },
1448 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1449 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1450 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1451 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1452 .moreElementsToNextPow2(TypeIdx: 1)
1453 .scalarize(TypeIdx: 1)
1454 .lower();
1455
1456 // For fmul reductions we need to split up into individual operations. We
1457 // clamp to 128 bit vectors then to 64bit vectors to produce a cascade of
1458 // smaller types, followed by scalarizing what remains.
1459 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FMUL)
1460 .widenScalarIf(
1461 Predicate: [HasFP16](const LegalityQuery &Query) {
1462 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1463 Query.Types[0].getScalarType().isBFloat16();
1464 },
1465 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1466 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1467 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1468 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1469 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1470 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1471 .scalarize(TypeIdx: 1)
1472 .lower();
1473
1474 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
1475 .scalarize(TypeIdx: 2)
1476 .lower();
1477
1478 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_ADD)
1479 .legalFor(Types: {{i8, v8i8},
1480 {i8, v16i8},
1481 {i16, v4i16},
1482 {i16, v8i16},
1483 {i32, v2i32},
1484 {i32, v4i32},
1485 {i64, v2i64}})
1486 .moreElementsToNextPow2(TypeIdx: 1)
1487 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1488 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1489 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1490 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1491 .widenVectorEltsToVectorMinSize(TypeIdx: 1, VectorSize: 64)
1492 .scalarize(TypeIdx: 1);
1493
1494 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
1495 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1496 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1497 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1498 .widenScalarIf(
1499 Predicate: [HasFP16](const LegalityQuery &Query) {
1500 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1501 Query.Types[0].getScalarType().isBFloat16();
1502 },
1503 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1504 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1505 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1506 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1507 .scalarize(TypeIdx: 1)
1508 .lower();
1509
1510 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_MUL)
1511 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1512 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1513 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
1514 .scalarize(TypeIdx: 1)
1515 .lower();
1516
1517 getActionDefinitionsBuilder(
1518 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1519 .legalFor(Types: {{i8, v8i8},
1520 {i8, v16i8},
1521 {i16, v4i16},
1522 {i16, v8i16},
1523 {i32, v2i32},
1524 {i32, v4i32}})
1525 .moreElementsIf(
1526 Predicate: [=](const LegalityQuery &Query) {
1527 return Query.Types[1].isVector() &&
1528 Query.Types[1].getElementType() != s8 &&
1529 Query.Types[1].getNumElements() & 1;
1530 },
1531 Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx: 1))
1532 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1533 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1534 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1535 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1536 .scalarize(TypeIdx: 1)
1537 .lower();
1538
1539 getActionDefinitionsBuilder(
1540 Opcodes: {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1541 // Try to break down into smaller vectors as long as they're at least 64
1542 // bits. This lets us use vector operations for some parts of the
1543 // reduction.
1544 .fewerElementsIf(
1545 Predicate: [=](const LegalityQuery &Q) {
1546 LLT SrcTy = Q.Types[1];
1547 if (SrcTy.isScalar())
1548 return false;
1549 if (!isPowerOf2_32(Value: SrcTy.getNumElements()))
1550 return false;
1551 // We can usually perform 64b vector operations.
1552 return SrcTy.getSizeInBits() > 64;
1553 },
1554 Mutation: [=](const LegalityQuery &Q) {
1555 LLT SrcTy = Q.Types[1];
1556 return std::make_pair(x: 1, y: SrcTy.divide(Factor: 2));
1557 })
1558 .scalarize(TypeIdx: 1)
1559 .lower();
1560
1561 // TODO: Update this to correct handling when adding AArch64/SVE support.
1562 getActionDefinitionsBuilder(Opcode: G_VECTOR_COMPRESS).lower();
1563
1564 // Access to floating-point environment.
1565 getActionDefinitionsBuilder(Opcodes: {G_GET_FPENV, G_SET_FPENV, G_RESET_FPENV,
1566 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1567 .libcall();
1568
1569 getActionDefinitionsBuilder(Opcodes: {G_GET_ROUNDING, G_SET_ROUNDING})
1570 .customFor(Types: {s32});
1571
1572 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).lower();
1573
1574 getActionDefinitionsBuilder(Opcode: G_PREFETCH).custom();
1575
1576 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
1577
1578 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
1579 .alwaysLegal();
1580 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
1581 getActionDefinitionsBuilder(Opcode: G_INVOKE_REGION_START).alwaysLegal();
1582
1583 verify(MII: *ST.getInstrInfo());
1584}
1585
1586bool AArch64LegalizerInfo::legalizeCustom(
1587 LegalizerHelper &Helper, MachineInstr &MI,
1588 LostDebugLocObserver &LocObserver) const {
1589 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1590 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1591 GISelChangeObserver &Observer = Helper.Observer;
1592 switch (MI.getOpcode()) {
1593 default:
1594 // No idea what to do.
1595 return false;
1596 case TargetOpcode::G_VAARG:
1597 return legalizeVaArg(MI, MRI, MIRBuilder);
1598 case TargetOpcode::G_LOAD:
1599 case TargetOpcode::G_STORE:
1600 return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
1601 case TargetOpcode::G_SHL:
1602 case TargetOpcode::G_ASHR:
1603 case TargetOpcode::G_LSHR:
1604 return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
1605 case TargetOpcode::G_GLOBAL_VALUE:
1606 return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
1607 case TargetOpcode::G_SBFX:
1608 case TargetOpcode::G_UBFX:
1609 return legalizeBitfieldExtract(MI, MRI, Helper);
1610 case TargetOpcode::G_FSHL:
1611 case TargetOpcode::G_FSHR:
1612 return legalizeFunnelShift(MI, MRI, MIRBuilder, Observer, Helper);
1613 case TargetOpcode::G_ROTR:
1614 return legalizeRotate(MI, MRI, Helper);
1615 case TargetOpcode::G_CTPOP:
1616 return legalizeCTPOP(MI, MRI, Helper);
1617 case TargetOpcode::G_ATOMIC_CMPXCHG:
1618 return legalizeAtomicCmpxchg128(MI, MRI, Helper);
1619 case TargetOpcode::G_CTTZ:
1620 return legalizeCTTZ(MI, Helper);
1621 case TargetOpcode::G_BZERO:
1622 case TargetOpcode::G_MEMCPY:
1623 case TargetOpcode::G_MEMMOVE:
1624 case TargetOpcode::G_MEMSET:
1625 case TargetOpcode::G_MEMSET_INLINE:
1626 return legalizeMemOps(MI, Helper);
1627 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1628 return legalizeExtractVectorElt(MI, MRI, Helper);
1629 case TargetOpcode::G_DYN_STACKALLOC:
1630 return legalizeDynStackAlloc(MI, Helper);
1631 case TargetOpcode::G_PREFETCH:
1632 return legalizePrefetch(MI, Helper);
1633 case TargetOpcode::G_ABS:
1634 return Helper.lowerAbsToCNeg(MI);
1635 case TargetOpcode::G_ICMP:
1636 return legalizeICMP(MI, MRI, MIRBuilder);
1637 case TargetOpcode::G_BITCAST:
1638 return legalizeBitcast(MI, Helper);
1639 case TargetOpcode::G_CONCAT_VECTORS:
1640 return legalizeConcatVectors(MI, MRI, MIRBuilder);
1641 case TargetOpcode::G_FPTRUNC:
1642 // In order to lower f16 to f64 properly, we need to use f32 as an
1643 // intermediary
1644 return legalizeFptrunc(MI, MIRBuilder, MRI);
1645 case TargetOpcode::G_GET_ROUNDING:
1646 return legalizeGetRounding(MI, MIRBuilder, MRI, Helper);
1647 case TargetOpcode::G_SET_ROUNDING:
1648 return legalizeSetRounding(MI, MIRBuilder, MRI, Helper);
1649 }
1650
1651 llvm_unreachable("expected switch to return");
1652}
1653
1654bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
1655 LegalizerHelper &Helper) const {
1656 assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
1657 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1658 // We're trying to handle casts from i1 vectors to scalars but reloading from
1659 // stack.
1660 if (!DstTy.isScalar() || !SrcTy.isVector() ||
1661 SrcTy.getElementType() != LLT::scalar(SizeInBits: 1))
1662 return false;
1663
1664 Helper.createStackStoreLoad(Res: DstReg, Val: SrcReg);
1665 MI.eraseFromParent();
1666 return true;
1667}
1668
1669bool AArch64LegalizerInfo::legalizeFunnelShift(MachineInstr &MI,
1670 MachineRegisterInfo &MRI,
1671 MachineIRBuilder &MIRBuilder,
1672 GISelChangeObserver &Observer,
1673 LegalizerHelper &Helper) const {
1674 assert(MI.getOpcode() == TargetOpcode::G_FSHL ||
1675 MI.getOpcode() == TargetOpcode::G_FSHR);
1676
1677 // Keep as G_FSHR if shift amount is a G_CONSTANT, else use generic
1678 // lowering
1679 Register ShiftNo = MI.getOperand(i: 3).getReg();
1680 LLT ShiftTy = MRI.getType(Reg: ShiftNo);
1681 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ShiftNo, MRI);
1682
1683 // Adjust shift amount according to Opcode (FSHL/FSHR)
1684 // Convert FSHL to FSHR
1685 LLT OperationTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1686 APInt BitWidth(ShiftTy.getSizeInBits(), OperationTy.getSizeInBits(), false);
1687
1688 // Lower non-constant shifts and leave zero shifts to the optimizer.
1689 if (!VRegAndVal || VRegAndVal->Value.urem(RHS: BitWidth) == 0)
1690 return (Helper.lowerFunnelShiftAsShifts(MI) ==
1691 LegalizerHelper::LegalizeResult::Legalized);
1692
1693 APInt Amount = VRegAndVal->Value.urem(RHS: BitWidth);
1694
1695 Amount = MI.getOpcode() == TargetOpcode::G_FSHL ? BitWidth - Amount : Amount;
1696
1697 // If the instruction is G_FSHR, has a 64-bit G_CONSTANT for shift amount
1698 // in the range of 0 <-> BitWidth, it is legal
1699 if (ShiftTy.getSizeInBits() == 64 && MI.getOpcode() == TargetOpcode::G_FSHR &&
1700 VRegAndVal->Value.ult(RHS: BitWidth))
1701 return true;
1702
1703 // Cast the ShiftNumber to a 64-bit type
1704 auto Cast64 = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Amount.zext(width: 64));
1705
1706 if (MI.getOpcode() == TargetOpcode::G_FSHR) {
1707 Observer.changingInstr(MI);
1708 MI.getOperand(i: 3).setReg(Cast64.getReg(Idx: 0));
1709 Observer.changedInstr(MI);
1710 }
1711 // If Opcode is FSHL, remove the FSHL instruction and create a FSHR
1712 // instruction
1713 else if (MI.getOpcode() == TargetOpcode::G_FSHL) {
1714 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FSHR, DstOps: {MI.getOperand(i: 0).getReg()},
1715 SrcOps: {MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getReg(),
1716 Cast64.getReg(Idx: 0)});
1717 MI.eraseFromParent();
1718 }
1719 return true;
1720}
1721
1722bool AArch64LegalizerInfo::legalizeICMP(MachineInstr &MI,
1723 MachineRegisterInfo &MRI,
1724 MachineIRBuilder &MIRBuilder) const {
1725 Register DstReg = MI.getOperand(i: 0).getReg();
1726 Register SrcReg1 = MI.getOperand(i: 2).getReg();
1727 Register SrcReg2 = MI.getOperand(i: 3).getReg();
1728 LLT DstTy = MRI.getType(Reg: DstReg);
1729 LLT SrcTy = MRI.getType(Reg: SrcReg1);
1730
1731 // Check the vector types are legal
1732 if (DstTy.getScalarSizeInBits() != SrcTy.getScalarSizeInBits() ||
1733 DstTy.getNumElements() != SrcTy.getNumElements() ||
1734 (DstTy.getSizeInBits() != 64 && DstTy.getSizeInBits() != 128))
1735 return false;
1736
1737 // Lowers G_ICMP NE => G_ICMP EQ to allow better pattern matching for
1738 // following passes
1739 CmpInst::Predicate Pred = (CmpInst::Predicate)MI.getOperand(i: 1).getPredicate();
1740 if (Pred != CmpInst::ICMP_NE)
1741 return true;
1742 Register CmpReg =
1743 MIRBuilder
1744 .buildICmp(Pred: CmpInst::ICMP_EQ, Res: MRI.getType(Reg: DstReg), Op0: SrcReg1, Op1: SrcReg2)
1745 .getReg(Idx: 0);
1746 MIRBuilder.buildNot(Dst: DstReg, Src0: CmpReg);
1747
1748 MI.eraseFromParent();
1749 return true;
1750}
1751
1752bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
1753 MachineRegisterInfo &MRI,
1754 LegalizerHelper &Helper) const {
1755 // To allow for imported patterns to match, we ensure that the rotate amount
1756 // is 64b with an extension.
1757 Register AmtReg = MI.getOperand(i: 2).getReg();
1758 LLT AmtTy = MRI.getType(Reg: AmtReg);
1759 (void)AmtTy;
1760 assert(AmtTy.isScalar() && "Expected a scalar rotate");
1761 assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
1762 auto NewAmt = Helper.MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 64), Op: AmtReg);
1763 Helper.Observer.changingInstr(MI);
1764 MI.getOperand(i: 2).setReg(NewAmt.getReg(Idx: 0));
1765 Helper.Observer.changedInstr(MI);
1766 return true;
1767}
1768
1769bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1770 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
1771 GISelChangeObserver &Observer) const {
1772 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1773 // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
1774 // G_ADD_LOW instructions.
1775 // By splitting this here, we can optimize accesses in the small code model by
1776 // folding in the G_ADD_LOW into the load/store offset.
1777 auto &GlobalOp = MI.getOperand(i: 1);
1778 // Don't modify an intrinsic call.
1779 if (GlobalOp.isSymbol())
1780 return true;
1781 const auto* GV = GlobalOp.getGlobal();
1782 if (GV->isThreadLocal())
1783 return true; // Don't want to modify TLS vars.
1784
1785 auto &TM = ST->getTargetLowering()->getTargetMachine();
1786 unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
1787
1788 if (OpFlags & AArch64II::MO_GOT)
1789 return true;
1790
1791 auto Offset = GlobalOp.getOffset();
1792 Register DstReg = MI.getOperand(i: 0).getReg();
1793 auto ADRP = MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {})
1794 .addGlobalAddress(GV, Offset, TargetFlags: OpFlags | AArch64II::MO_PAGE);
1795 // Set the regclass on the dest reg too.
1796 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1797
1798 // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
1799 // by creating a MOVK that sets bits 48-63 of the register to (global address
1800 // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
1801 // prevent an incorrect tag being generated during relocation when the
1802 // global appears before the code section. Without the offset, a global at
1803 // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
1804 // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
1805 // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
1806 // instead of `0xf`.
1807 // This assumes that we're in the small code model so we can assume a binary
1808 // size of <= 4GB, which makes the untagged PC relative offset positive. The
1809 // binary must also be loaded into address range [0, 2^48). Both of these
1810 // properties need to be ensured at runtime when using tagged addresses.
1811 if (OpFlags & AArch64II::MO_TAGGED) {
1812 assert(!Offset &&
1813 "Should not have folded in an offset for a tagged global!");
1814 ADRP = MIRBuilder.buildInstr(Opc: AArch64::MOVKXi, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {ADRP})
1815 .addGlobalAddress(GV, Offset: 0x100000000,
1816 TargetFlags: AArch64II::MO_PREL | AArch64II::MO_G3)
1817 .addImm(Val: 48);
1818 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1819 }
1820
1821 MIRBuilder.buildInstr(Opc: AArch64::G_ADD_LOW, DstOps: {DstReg}, SrcOps: {ADRP})
1822 .addGlobalAddress(GV, Offset,
1823 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1824 MI.eraseFromParent();
1825 return true;
1826}
1827
1828bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1829 MachineInstr &MI) const {
1830 MachineIRBuilder &MIB = Helper.MIRBuilder;
1831 MachineRegisterInfo &MRI = *MIB.getMRI();
1832
1833 auto LowerUnaryOp = [&MI, &MIB](unsigned Opcode) {
1834 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1835 MI.eraseFromParent();
1836 return true;
1837 };
1838 auto LowerBinOp = [&MI, &MIB](unsigned Opcode) {
1839 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1840 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
1841 MI.eraseFromParent();
1842 return true;
1843 };
1844 auto LowerTriOp = [&MI, &MIB](unsigned Opcode) {
1845 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1846 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4)});
1847 MI.eraseFromParent();
1848 return true;
1849 };
1850
1851 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1852 switch (IntrinsicID) {
1853 case Intrinsic::vacopy: {
1854 MachineFunction &MF = *MI.getMF();
1855 const DataLayout &DL = MF.getDataLayout();
1856 unsigned PtrSize = DL.getPointerSize();
1857 unsigned VaListSize = ST->getTargetLowering()->getVaListSizeInBits(DL) / 8;
1858 auto Val = MF.getRegInfo().createGenericVirtualRegister(
1859 Ty: LLT::integer(SizeInBits: VaListSize * 8));
1860 MIB.buildLoad(Res: Val, Addr: MI.getOperand(i: 2),
1861 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1862 F: MachineMemOperand::MOLoad,
1863 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1864 MIB.buildStore(Val, Addr: MI.getOperand(i: 1),
1865 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1866 F: MachineMemOperand::MOStore,
1867 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1868 MI.eraseFromParent();
1869 return true;
1870 }
1871 case Intrinsic::get_dynamic_area_offset: {
1872 MIB.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: 0);
1873 MI.eraseFromParent();
1874 return true;
1875 }
1876 case Intrinsic::aarch64_mops_memset_tag: {
1877 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1878 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
1879 // the instruction).
1880 auto &Value = MI.getOperand(i: 3);
1881 Register ExtValueReg = MIB.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
1882 Value.setReg(ExtValueReg);
1883 return true;
1884 }
1885 case Intrinsic::aarch64_prefetch: {
1886 auto &AddrVal = MI.getOperand(i: 1);
1887
1888 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1889 int64_t Target = MI.getOperand(i: 3).getImm();
1890 int64_t IsStream = MI.getOperand(i: 4).getImm();
1891 int64_t IsData = MI.getOperand(i: 5).getImm();
1892
1893 unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
1894 (!IsData << 3) | // IsDataCache bit
1895 (Target << 1) | // Cache level bits
1896 (unsigned)IsStream; // Stream bit
1897
1898 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
1899 MI.eraseFromParent();
1900 return true;
1901 }
1902 case Intrinsic::aarch64_range_prefetch: {
1903 auto &AddrVal = MI.getOperand(i: 1);
1904
1905 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1906 int64_t IsStream = MI.getOperand(i: 3).getImm();
1907 unsigned PrfOp = (IsStream << 2) | IsWrite;
1908
1909 MIB.buildInstr(Opcode: AArch64::G_AARCH64_RANGE_PREFETCH)
1910 .addImm(Val: PrfOp)
1911 .add(MO: AddrVal)
1912 .addUse(RegNo: MI.getOperand(i: 4).getReg()); // Metadata
1913 MI.eraseFromParent();
1914 return true;
1915 }
1916 case Intrinsic::aarch64_prefetch_ir: {
1917 auto &AddrVal = MI.getOperand(i: 1);
1918 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: 24).add(MO: AddrVal);
1919 MI.eraseFromParent();
1920 return true;
1921 }
1922 case Intrinsic::aarch64_neon_uaddv:
1923 case Intrinsic::aarch64_neon_saddv:
1924 case Intrinsic::aarch64_neon_umaxv:
1925 case Intrinsic::aarch64_neon_smaxv:
1926 case Intrinsic::aarch64_neon_uminv:
1927 case Intrinsic::aarch64_neon_sminv: {
1928 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1929 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1930 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1931
1932 auto OldDst = MI.getOperand(i: 0).getReg();
1933 auto OldDstTy = MRI.getType(Reg: OldDst);
1934 LLT NewDstTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType();
1935 if (OldDstTy == NewDstTy)
1936 return true;
1937
1938 auto NewDst = MRI.createGenericVirtualRegister(Ty: NewDstTy);
1939
1940 Helper.Observer.changingInstr(MI);
1941 MI.getOperand(i: 0).setReg(NewDst);
1942 Helper.Observer.changedInstr(MI);
1943
1944 MIB.setInsertPt(MBB&: MIB.getMBB(), II: ++MIB.getInsertPt());
1945 MIB.buildExtOrTrunc(ExtOpc: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1946 Res: OldDst, Op: NewDst);
1947
1948 return true;
1949 }
1950 case Intrinsic::aarch64_neon_uaddlp:
1951 case Intrinsic::aarch64_neon_saddlp: {
1952 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1953 ? AArch64::G_UADDLP
1954 : AArch64::G_SADDLP;
1955 MIB.buildInstr(Opc, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1956 MI.eraseFromParent();
1957
1958 return true;
1959 }
1960 case Intrinsic::aarch64_neon_uaddlv:
1961 case Intrinsic::aarch64_neon_saddlv: {
1962 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1963 ? AArch64::G_UADDLV
1964 : AArch64::G_SADDLV;
1965 Register DstReg = MI.getOperand(i: 0).getReg();
1966 Register SrcReg = MI.getOperand(i: 2).getReg();
1967 LLT DstTy = MRI.getType(Reg: DstReg);
1968
1969 LLT MidTy, ExtTy;
1970 if (DstTy.isScalar() && DstTy.getScalarSizeInBits() <= 32) {
1971 ExtTy = LLT::integer(SizeInBits: 32);
1972 MidTy = LLT::fixed_vector(NumElements: 4, ScalarTy: ExtTy);
1973 } else {
1974 ExtTy = LLT::integer(SizeInBits: 64);
1975 MidTy = LLT::fixed_vector(NumElements: 2, ScalarTy: ExtTy);
1976 }
1977
1978 Register MidReg =
1979 MIB.buildInstr(Opc, DstOps: {MidTy}, SrcOps: {SrcReg})->getOperand(i: 0).getReg();
1980 Register ZeroReg =
1981 MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0)->getOperand(i: 0).getReg();
1982 Register ExtReg = MIB.buildInstr(Opc: AArch64::G_EXTRACT_VECTOR_ELT, DstOps: {ExtTy},
1983 SrcOps: {MidReg, ZeroReg})
1984 .getReg(Idx: 0);
1985
1986 if (DstTy.getScalarSizeInBits() < 32)
1987 MIB.buildTrunc(Res: DstReg, Op: ExtReg);
1988 else
1989 MIB.buildCopy(Res: DstReg, Op: ExtReg);
1990
1991 MI.eraseFromParent();
1992
1993 return true;
1994 }
1995 case Intrinsic::aarch64_neon_fmax:
1996 return LowerBinOp(TargetOpcode::G_FMAXIMUM);
1997 case Intrinsic::aarch64_neon_fmin:
1998 return LowerBinOp(TargetOpcode::G_FMINIMUM);
1999 case Intrinsic::aarch64_neon_fmaxnm:
2000 return LowerBinOp(TargetOpcode::G_FMAXNUM);
2001 case Intrinsic::aarch64_neon_fminnm:
2002 return LowerBinOp(TargetOpcode::G_FMINNUM);
2003 case Intrinsic::aarch64_neon_pmul:
2004 return LowerBinOp(TargetOpcode::G_CLMUL);
2005 case Intrinsic::aarch64_neon_pmull:
2006 case Intrinsic::aarch64_neon_pmull64:
2007 return LowerBinOp(AArch64::G_PMULL);
2008 case Intrinsic::aarch64_neon_smull:
2009 return LowerBinOp(AArch64::G_SMULL);
2010 case Intrinsic::aarch64_neon_umull:
2011 return LowerBinOp(AArch64::G_UMULL);
2012 case Intrinsic::aarch64_neon_sabd:
2013 return LowerBinOp(TargetOpcode::G_ABDS);
2014 case Intrinsic::aarch64_neon_uabd:
2015 return LowerBinOp(TargetOpcode::G_ABDU);
2016 case Intrinsic::aarch64_neon_uhadd:
2017 return LowerBinOp(TargetOpcode::G_UAVGFLOOR);
2018 case Intrinsic::aarch64_neon_urhadd:
2019 return LowerBinOp(TargetOpcode::G_UAVGCEIL);
2020 case Intrinsic::aarch64_neon_shadd:
2021 return LowerBinOp(TargetOpcode::G_SAVGFLOOR);
2022 case Intrinsic::aarch64_neon_srhadd:
2023 return LowerBinOp(TargetOpcode::G_SAVGCEIL);
2024 case Intrinsic::aarch64_neon_sqshrn: {
2025 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2026 return true;
2027 // Create right shift instruction. Store the output register in Shr.
2028 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2029 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2030 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2031 // Build the narrow intrinsic, taking in Shr.
2032 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2033 MI.eraseFromParent();
2034 return true;
2035 }
2036 case Intrinsic::aarch64_neon_sqshrun: {
2037 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2038 return true;
2039 // Create right shift instruction. Store the output register in Shr.
2040 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2041 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2042 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2043 // Build the narrow intrinsic, taking in Shr.
2044 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2045 MI.eraseFromParent();
2046 return true;
2047 }
2048 case Intrinsic::aarch64_neon_sqrshrn: {
2049 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2050 return true;
2051 // Create right shift instruction. Store the output register in Shr.
2052 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2053 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2054 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2055 // Build the narrow intrinsic, taking in Shr.
2056 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2057 MI.eraseFromParent();
2058 return true;
2059 }
2060 case Intrinsic::aarch64_neon_sqrshrun: {
2061 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2062 return true;
2063 // Create right shift instruction. Store the output register in Shr.
2064 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2065 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2066 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2067 // Build the narrow intrinsic, taking in Shr.
2068 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2069 MI.eraseFromParent();
2070 return true;
2071 }
2072 case Intrinsic::aarch64_neon_uqrshrn: {
2073 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2074 return true;
2075 // Create right shift instruction. Store the output register in Shr.
2076 auto Shr = MIB.buildInstr(Opc: AArch64::G_URSHR_I,
2077 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2078 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2079 // Build the narrow intrinsic, taking in Shr.
2080 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2081 MI.eraseFromParent();
2082 return true;
2083 }
2084 case Intrinsic::aarch64_neon_uqshrn: {
2085 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2086 return true;
2087 // Create right shift instruction. Store the output register in Shr.
2088 auto Shr = MIB.buildInstr(Opc: AArch64::G_VLSHR,
2089 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2090 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2091 // Build the narrow intrinsic, taking in Shr.
2092 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2093 MI.eraseFromParent();
2094 return true;
2095 }
2096 case Intrinsic::aarch64_neon_sqshlu: {
2097 // Check if last operand is constant vector dup
2098 auto ShiftAmount =
2099 isConstantOrConstantSplatVector(Def: MI.getOperand(i: 3).getReg(), MRI);
2100 if (ShiftAmount) {
2101 // If so, create a new intrinsic with the correct shift amount
2102 MIB.buildInstr(Opc: AArch64::G_SQSHLU_I, DstOps: {MI.getOperand(i: 0)},
2103 SrcOps: {MI.getOperand(i: 2)})
2104 .addImm(Val: ShiftAmount->getSExtValue());
2105 MI.eraseFromParent();
2106 return true;
2107 }
2108 return false;
2109 }
2110 case Intrinsic::aarch64_neon_vsli: {
2111 MIB.buildInstr(
2112 Opc: AArch64::G_SLI, DstOps: {MI.getOperand(i: 0)},
2113 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2114 MI.eraseFromParent();
2115 break;
2116 }
2117 case Intrinsic::aarch64_neon_vsri: {
2118 MIB.buildInstr(
2119 Opc: AArch64::G_SRI, DstOps: {MI.getOperand(i: 0)},
2120 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2121 MI.eraseFromParent();
2122 break;
2123 }
2124 case Intrinsic::aarch64_neon_abs: {
2125 // Lower the intrinsic to G_ABS.
2126 MIB.buildInstr(Opc: TargetOpcode::G_ABS, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
2127 MI.eraseFromParent();
2128 return true;
2129 }
2130 case Intrinsic::aarch64_neon_addhn:
2131 return LowerBinOp(AArch64::G_ADDHN);
2132 case Intrinsic::aarch64_neon_sqadd: {
2133 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2134 return LowerBinOp(TargetOpcode::G_SADDSAT);
2135 break;
2136 }
2137 case Intrinsic::aarch64_neon_sqsub: {
2138 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2139 return LowerBinOp(TargetOpcode::G_SSUBSAT);
2140 break;
2141 }
2142 case Intrinsic::aarch64_neon_uqadd: {
2143 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2144 return LowerBinOp(TargetOpcode::G_UADDSAT);
2145 break;
2146 }
2147 case Intrinsic::aarch64_neon_uqsub: {
2148 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2149 return LowerBinOp(TargetOpcode::G_USUBSAT);
2150 break;
2151 }
2152 case Intrinsic::aarch64_neon_udot:
2153 return LowerTriOp(AArch64::G_UDOT);
2154 case Intrinsic::aarch64_neon_sdot:
2155 return LowerTriOp(AArch64::G_SDOT);
2156 case Intrinsic::aarch64_neon_usdot:
2157 return LowerTriOp(AArch64::G_USDOT);
2158 case Intrinsic::aarch64_neon_sqxtn:
2159 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_S);
2160 case Intrinsic::aarch64_neon_sqxtun:
2161 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_U);
2162 case Intrinsic::aarch64_neon_uqxtn:
2163 return LowerUnaryOp(TargetOpcode::G_TRUNC_USAT_U);
2164 case Intrinsic::aarch64_neon_fcvtzu:
2165 return LowerUnaryOp(TargetOpcode::G_FPTOUI_SAT);
2166 case Intrinsic::aarch64_neon_fcvtzs:
2167 return LowerUnaryOp(TargetOpcode::G_FPTOSI_SAT);
2168 case Intrinsic::aarch64_neon_cls:
2169 return LowerUnaryOp(TargetOpcode::G_CTLS);
2170
2171 case Intrinsic::vector_reverse:
2172 // TODO: Add support for vector_reverse
2173 return false;
2174 }
2175
2176 return true;
2177}
2178
2179bool AArch64LegalizerInfo::legalizeShlAshrLshr(
2180 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2181 GISelChangeObserver &Observer) const {
2182 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
2183 MI.getOpcode() == TargetOpcode::G_LSHR ||
2184 MI.getOpcode() == TargetOpcode::G_SHL);
2185 // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
2186 // imported patterns can select it later. Either way, it will be legal.
2187 Register AmtReg = MI.getOperand(i: 2).getReg();
2188 LLT AmtRegEltTy = MRI.getType(Reg: AmtReg).getScalarType();
2189 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI);
2190 if (!VRegAndVal)
2191 return true;
2192 // Check the shift amount is in range for an immediate form.
2193 int64_t Amount = VRegAndVal->Value.getSExtValue();
2194 if (Amount > 31)
2195 return true; // This will have to remain a register variant.
2196 auto ExtCst =
2197 MIRBuilder.buildConstant(Res: AmtRegEltTy.changeElementSize(NewEltSize: 64), Val: Amount);
2198 Observer.changingInstr(MI);
2199 MI.getOperand(i: 2).setReg(ExtCst.getReg(Idx: 0));
2200 Observer.changedInstr(MI);
2201 return true;
2202}
2203
2204static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset,
2205 MachineRegisterInfo &MRI) {
2206 Base = Root;
2207 Offset = 0;
2208
2209 Register NewBase;
2210 int64_t NewOffset;
2211 if (mi_match(R: Root, MRI, P: m_GPtrAdd(L: m_Reg(R&: NewBase), R: m_ICst(Cst&: NewOffset))) &&
2212 isShiftedInt<7, 3>(x: NewOffset)) {
2213 Base = NewBase;
2214 Offset = NewOffset;
2215 }
2216}
2217
2218// FIXME: This should be removed and replaced with the generic bitcast legalize
2219// action.
2220bool AArch64LegalizerInfo::legalizeLoadStore(
2221 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2222 GISelChangeObserver &Observer) const {
2223 assert(MI.getOpcode() == TargetOpcode::G_STORE ||
2224 MI.getOpcode() == TargetOpcode::G_LOAD);
2225 // Here we just try to handle vector loads/stores where our value type might
2226 // have pointer elements, which the SelectionDAG importer can't handle. To
2227 // allow the existing patterns for s64 to fire for p0, we just try to bitcast
2228 // the value to use s64 types.
2229
2230 // Custom legalization requires the instruction, if not deleted, must be fully
2231 // legalized. In order to allow further legalization of the inst, we create
2232 // a new instruction and erase the existing one.
2233
2234 Register ValReg = MI.getOperand(i: 0).getReg();
2235 const LLT ValTy = MRI.getType(Reg: ValReg);
2236
2237 if (ValTy == LLT::scalar(SizeInBits: 128)) {
2238
2239 AtomicOrdering Ordering = (*MI.memoperands_begin())->getSuccessOrdering();
2240 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2241 bool IsLoadAcquire = IsLoad && Ordering == AtomicOrdering::Acquire;
2242 bool IsStoreRelease = !IsLoad && isReleaseOrStronger(AO: Ordering);
2243 bool IsRcpC3 =
2244 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
2245 bool IsLSCP = ST->hasLSE2() && ST->hasLSCP() && IsStoreRelease;
2246
2247 LLT s64 = LLT::integer(SizeInBits: 64);
2248
2249 unsigned Opcode;
2250 if (IsLSCP) {
2251 Opcode = AArch64::STLPi;
2252 } else if (IsRcpC3) {
2253 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
2254 } else {
2255 // For LSE2, loads/stores should have been converted to monotonic and had
2256 // a fence inserted after them.
2257 assert(Ordering == AtomicOrdering::Monotonic ||
2258 Ordering == AtomicOrdering::Unordered);
2259 assert(ST->hasLSE2() && "ldp/stp not single copy atomic without +lse2");
2260
2261 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
2262 }
2263
2264 MachineInstrBuilder NewI;
2265 if (IsLoad) {
2266 NewI = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {s64, s64}, SrcOps: {});
2267 MIRBuilder.buildMergeLikeInstr(
2268 Res: ValReg, Ops: {NewI->getOperand(i: 0), NewI->getOperand(i: 1)});
2269 } else {
2270 auto Split = MIRBuilder.buildUnmerge(Res: s64, Op: MI.getOperand(i: 0));
2271 NewI = MIRBuilder.buildInstr(
2272 Opc: Opcode, DstOps: {}, SrcOps: {Split->getOperand(i: 0), Split->getOperand(i: 1)});
2273 }
2274
2275 if (IsRcpC3 || IsLSCP) {
2276 NewI.addUse(RegNo: MI.getOperand(i: 1).getReg());
2277 } else {
2278 Register Base;
2279 int Offset;
2280 matchLDPSTPAddrMode(Root: MI.getOperand(i: 1).getReg(), Base, Offset, MRI);
2281 NewI.addUse(RegNo: Base);
2282 NewI.addImm(Val: Offset / 8);
2283 }
2284
2285 NewI.cloneMemRefs(OtherMI: MI);
2286 constrainSelectedInstRegOperands(I&: *NewI, TII: *ST->getInstrInfo(),
2287 TRI: *MRI.getTargetRegisterInfo(),
2288 RBI: *ST->getRegBankInfo());
2289 MI.eraseFromParent();
2290 return true;
2291 }
2292
2293 if (!ValTy.isPointerVector() ||
2294 ValTy.getElementType().getAddressSpace() != 0) {
2295 LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
2296 return false;
2297 }
2298
2299 unsigned PtrSize = ValTy.getElementType().getSizeInBits();
2300 const LLT NewTy = LLT::vector(EC: ValTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: PtrSize));
2301 auto &MMO = **MI.memoperands_begin();
2302 MMO.setType(NewTy);
2303
2304 if (MI.getOpcode() == TargetOpcode::G_STORE) {
2305 auto Bitcast = MIRBuilder.buildBitcast(Dst: NewTy, Src: ValReg);
2306 MIRBuilder.buildStore(Val: Bitcast.getReg(Idx: 0), Addr: MI.getOperand(i: 1), MMO);
2307 } else {
2308 auto NewLoad = MIRBuilder.buildLoad(Res: NewTy, Addr: MI.getOperand(i: 1), MMO);
2309 MIRBuilder.buildBitcast(Dst: ValReg, Src: NewLoad);
2310 }
2311 MI.eraseFromParent();
2312 return true;
2313}
2314
2315bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
2316 MachineRegisterInfo &MRI,
2317 MachineIRBuilder &MIRBuilder) const {
2318 MachineFunction &MF = MIRBuilder.getMF();
2319 Align Alignment(MI.getOperand(i: 2).getImm());
2320 Register Dst = MI.getOperand(i: 0).getReg();
2321 Register ListPtr = MI.getOperand(i: 1).getReg();
2322
2323 LLT PtrTy = MRI.getType(Reg: ListPtr);
2324 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
2325
2326 const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
2327 const Align PtrAlign = Align(PtrSize);
2328 auto List = MIRBuilder.buildLoad(
2329 Res: PtrTy, Addr: ListPtr,
2330 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2331 MemTy: PtrTy, BaseAlignment: PtrAlign));
2332
2333 MachineInstrBuilder DstPtr;
2334 if (Alignment > PtrAlign) {
2335 // Realign the list to the actual required alignment.
2336 auto AlignMinus1 =
2337 MIRBuilder.buildConstant(Res: IntPtrTy, Val: Alignment.value() - 1);
2338 auto ListTmp = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: List, Op1: AlignMinus1.getReg(Idx: 0));
2339 DstPtr = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: ListTmp, NumBits: Log2(A: Alignment));
2340 } else
2341 DstPtr = List;
2342
2343 LLT ValTy = MRI.getType(Reg: Dst);
2344 uint64_t ValSize = ValTy.getSizeInBits() / 8;
2345 MIRBuilder.buildLoad(
2346 Res: Dst, Addr: DstPtr,
2347 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2348 MemTy: ValTy, BaseAlignment: std::max(a: Alignment, b: PtrAlign)));
2349
2350 auto Size = MIRBuilder.buildConstant(Res: IntPtrTy, Val: alignTo(Size: ValSize, A: PtrAlign));
2351
2352 auto NewList = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: DstPtr, Op1: Size.getReg(Idx: 0));
2353
2354 MIRBuilder.buildStore(Val: NewList, Addr: ListPtr,
2355 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
2356 F: MachineMemOperand::MOStore,
2357 MemTy: PtrTy, BaseAlignment: PtrAlign));
2358
2359 MI.eraseFromParent();
2360 return true;
2361}
2362
2363bool AArch64LegalizerInfo::legalizeBitfieldExtract(
2364 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2365 // Only legal if we can select immediate forms.
2366 // TODO: Lower this otherwise.
2367 return getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI) &&
2368 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
2369}
2370
2371bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
2372 MachineRegisterInfo &MRI,
2373 LegalizerHelper &Helper) const {
2374 // When there is no integer popcount instruction (FEAT_CSSC isn't available),
2375 // it can be more efficiently lowered to the following sequence that uses
2376 // AdvSIMD registers/instructions as long as the copies to/from the AdvSIMD
2377 // registers are cheap.
2378 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
2379 // CNT V0.8B, V0.8B // 8xbyte pop-counts
2380 // ADDV B0, V0.8B // sum 8xbyte pop-counts
2381 // UMOV X0, V0.B[0] // copy byte result back to integer reg
2382 //
2383 // For 128 bit vector popcounts, we lower to the following sequence:
2384 // cnt.16b v0, v0 // v8s16, v4s32, v2s64
2385 // uaddlp.8h v0, v0 // v8s16, v4s32, v2s64
2386 // uaddlp.4s v0, v0 // v4s32, v2s64
2387 // uaddlp.2d v0, v0 // v2s64
2388 //
2389 // For 64 bit vector popcounts, we lower to the following sequence:
2390 // cnt.8b v0, v0 // v4s16, v2s32
2391 // uaddlp.4h v0, v0 // v4s16, v2s32
2392 // uaddlp.2s v0, v0 // v2s32
2393
2394 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2395 Register Dst = MI.getOperand(i: 0).getReg();
2396 Register Val = MI.getOperand(i: 1).getReg();
2397 LLT Ty = MRI.getType(Reg: Val);
2398
2399 LLT i64 = LLT::integer(SizeInBits: 64);
2400 LLT i32 = LLT::integer(SizeInBits: 32);
2401 LLT i16 = LLT::integer(SizeInBits: 16);
2402 LLT i8 = LLT::integer(SizeInBits: 8);
2403 unsigned Size = Ty.getSizeInBits();
2404
2405 assert(Ty == MRI.getType(Dst) &&
2406 "Expected src and dst to have the same type!");
2407
2408 if (ST->hasCSSC() && Ty.isScalar() && Size == 128) {
2409
2410 auto Split = MIRBuilder.buildUnmerge(Res: i64, Op: Val);
2411 auto CTPOP1 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 0));
2412 auto CTPOP2 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 1));
2413 auto Add = MIRBuilder.buildAdd(Dst: i64, Src0: CTPOP1, Src1: CTPOP2);
2414
2415 MIRBuilder.buildZExt(Res: Dst, Op: Add);
2416 MI.eraseFromParent();
2417 return true;
2418 }
2419
2420 if (!ST->hasNEON() ||
2421 MI.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
2422 // Use generic lowering when custom lowering is not possible.
2423 return Ty.isScalar() && (Size == 32 || Size == 64) &&
2424 Helper.lowerBitCount(MI) ==
2425 LegalizerHelper::LegalizeResult::Legalized;
2426 }
2427
2428 // Pre-conditioning: widen Val up to the nearest vector type.
2429 // s32,s64,v4s16,v2s32 -> v8i8
2430 // v8s16,v4s32,v2s64 -> v16i8
2431 LLT VTy = Size == 128 ? LLT::fixed_vector(NumElements: 16, ScalarTy: i8) : LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
2432 if (Ty.isScalar()) {
2433 assert((Size == 32 || Size == 64 || Size == 128) && "Expected only 32, 64, or 128 bit scalars!");
2434 if (Size == 32) {
2435 Val = MIRBuilder.buildZExt(Res: i64, Op: Val).getReg(Idx: 0);
2436 }
2437 }
2438 Val = MIRBuilder.buildBitcast(Dst: VTy, Src: Val).getReg(Idx: 0);
2439
2440 // Count bits in each byte-sized lane.
2441 auto CTPOP = MIRBuilder.buildCTPOP(Dst: VTy, Src0: Val);
2442
2443 // Sum across lanes.
2444 if (ST->hasDotProd() && Ty.isVector() && Ty.getNumElements() >= 2 &&
2445 Ty.getScalarSizeInBits() != 16) {
2446 LLT Dt = Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64) ? LLT::fixed_vector(NumElements: 4, ScalarTy: i32) : Ty;
2447 auto Zeros = MIRBuilder.buildConstant(Res: Dt, Val: 0);
2448 auto Ones = MIRBuilder.buildConstant(Res: VTy, Val: 1);
2449 MachineInstrBuilder Sum;
2450
2451 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2452 auto UDOT =
2453 MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2454 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UADDLP, DstOps: {Ty}, SrcOps: {UDOT});
2455 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2456 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2457 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2458 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2459 } else {
2460 llvm_unreachable("unexpected vector shape");
2461 }
2462
2463 Sum->getOperand(i: 0).setReg(Dst);
2464 MI.eraseFromParent();
2465 return true;
2466 }
2467
2468 Register HSum = CTPOP.getReg(Idx: 0);
2469 unsigned Opc;
2470 SmallVector<LLT> HAddTys;
2471 if (Ty.isScalar()) {
2472 Opc = Intrinsic::aarch64_neon_uaddlv;
2473 HAddTys.push_back(Elt: i32);
2474 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: i16)) {
2475 Opc = Intrinsic::aarch64_neon_uaddlp;
2476 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2477 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2478 Opc = Intrinsic::aarch64_neon_uaddlp;
2479 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2480 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2481 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2482 Opc = Intrinsic::aarch64_neon_uaddlp;
2483 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2484 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2485 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i64));
2486 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i16)) {
2487 Opc = Intrinsic::aarch64_neon_uaddlp;
2488 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2489 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2490 Opc = Intrinsic::aarch64_neon_uaddlp;
2491 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2492 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i32));
2493 } else
2494 llvm_unreachable("unexpected vector shape");
2495 MachineInstrBuilder UADD;
2496 for (LLT HTy : HAddTys) {
2497 UADD = MIRBuilder.buildIntrinsic(ID: Opc, Res: {HTy}).addUse(RegNo: HSum);
2498 HSum = UADD.getReg(Idx: 0);
2499 }
2500
2501 // Post-conditioning.
2502 if (Ty.isScalar() && (Size == 64 || Size == 128))
2503 MIRBuilder.buildZExt(Res: Dst, Op: UADD);
2504 else
2505 UADD->getOperand(i: 0).setReg(Dst);
2506 MI.eraseFromParent();
2507 return true;
2508}
2509
2510bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
2511 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2512 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2513 LLT i64 = LLT::integer(SizeInBits: 64);
2514 auto Addr = MI.getOperand(i: 1).getReg();
2515 auto DesiredI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 2));
2516 auto NewI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 3));
2517 auto DstLo = MRI.createGenericVirtualRegister(Ty: i64);
2518 auto DstHi = MRI.createGenericVirtualRegister(Ty: i64);
2519
2520 MachineInstrBuilder CAS;
2521 if (ST->hasLSE()) {
2522 // We have 128-bit CASP instructions taking XSeqPair registers, which are
2523 // s128. We need the merge/unmerge to bracket the expansion and pair up with
2524 // the rest of the MIR so we must reassemble the extracted registers into a
2525 // 128-bit known-regclass one with code like this:
2526 //
2527 // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
2528 // %out = CASP %in1, ...
2529 // %OldLo = G_EXTRACT %out, 0
2530 // %OldHi = G_EXTRACT %out, 64
2531 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2532 unsigned Opcode;
2533 switch (Ordering) {
2534 case AtomicOrdering::Acquire:
2535 Opcode = AArch64::CASPAX;
2536 break;
2537 case AtomicOrdering::Release:
2538 Opcode = AArch64::CASPLX;
2539 break;
2540 case AtomicOrdering::AcquireRelease:
2541 case AtomicOrdering::SequentiallyConsistent:
2542 Opcode = AArch64::CASPALX;
2543 break;
2544 default:
2545 Opcode = AArch64::CASPX;
2546 break;
2547 }
2548
2549 LLT s128 = LLT::integer(SizeInBits: 128);
2550 auto CASDst = MRI.createGenericVirtualRegister(Ty: s128);
2551 auto CASDesired = MRI.createGenericVirtualRegister(Ty: s128);
2552 auto CASNew = MRI.createGenericVirtualRegister(Ty: s128);
2553 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASDesired}, SrcOps: {})
2554 .addUse(RegNo: DesiredI->getOperand(i: 0).getReg())
2555 .addImm(Val: AArch64::sube64)
2556 .addUse(RegNo: DesiredI->getOperand(i: 1).getReg())
2557 .addImm(Val: AArch64::subo64);
2558 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASNew}, SrcOps: {})
2559 .addUse(RegNo: NewI->getOperand(i: 0).getReg())
2560 .addImm(Val: AArch64::sube64)
2561 .addUse(RegNo: NewI->getOperand(i: 1).getReg())
2562 .addImm(Val: AArch64::subo64);
2563
2564 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {CASDst}, SrcOps: {CASDesired, CASNew, Addr});
2565
2566 MIRBuilder.buildExtract(Res: {DstLo}, Src: {CASDst}, Index: 0);
2567 MIRBuilder.buildExtract(Res: {DstHi}, Src: {CASDst}, Index: 64);
2568 } else {
2569 // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
2570 // can take arbitrary registers so it just has the normal GPR64 operands the
2571 // rest of AArch64 is expecting.
2572 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2573 unsigned Opcode;
2574 switch (Ordering) {
2575 case AtomicOrdering::Acquire:
2576 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
2577 break;
2578 case AtomicOrdering::Release:
2579 Opcode = AArch64::CMP_SWAP_128_RELEASE;
2580 break;
2581 case AtomicOrdering::AcquireRelease:
2582 case AtomicOrdering::SequentiallyConsistent:
2583 Opcode = AArch64::CMP_SWAP_128;
2584 break;
2585 default:
2586 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
2587 break;
2588 }
2589
2590 auto Scratch = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2591 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {DstLo, DstHi, Scratch},
2592 SrcOps: {Addr, DesiredI->getOperand(i: 0),
2593 DesiredI->getOperand(i: 1), NewI->getOperand(i: 0),
2594 NewI->getOperand(i: 1)});
2595 }
2596
2597 CAS.cloneMemRefs(OtherMI: MI);
2598 constrainSelectedInstRegOperands(I&: *CAS, TII: *ST->getInstrInfo(),
2599 TRI: *MRI.getTargetRegisterInfo(),
2600 RBI: *ST->getRegBankInfo());
2601
2602 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {DstLo, DstHi});
2603 MI.eraseFromParent();
2604 return true;
2605}
2606
2607bool AArch64LegalizerInfo::legalizeCTTZ(MachineInstr &MI,
2608 LegalizerHelper &Helper) const {
2609 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2610 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2611 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2612 auto BitReverse = MIRBuilder.buildBitReverse(Dst: Ty, Src: MI.getOperand(i: 1));
2613 MIRBuilder.buildCTLZ(Dst: MI.getOperand(i: 0).getReg(), Src0: BitReverse);
2614 MI.eraseFromParent();
2615 return true;
2616}
2617
2618bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
2619 LegalizerHelper &Helper) const {
2620 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2621
2622 // Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
2623 if (MI.getOpcode() == TargetOpcode::G_MEMSET ||
2624 MI.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2625 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
2626 // the instruction).
2627 auto &Value = MI.getOperand(i: 1);
2628 Register ExtValueReg =
2629 MIRBuilder.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
2630 Value.setReg(ExtValueReg);
2631 return true;
2632 }
2633
2634 return false;
2635}
2636
2637bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2638 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2639 const GExtractVectorElement *Element = cast<GExtractVectorElement>(Val: &MI);
2640 auto VRegAndVal =
2641 getIConstantVRegValWithLookThrough(VReg: Element->getIndexReg(), MRI);
2642 if (VRegAndVal)
2643 return true;
2644 LLT VecTy = MRI.getType(Reg: Element->getVectorReg());
2645 if (VecTy.isScalableVector())
2646 return true;
2647 return Helper.lowerExtractInsertVectorElt(MI) !=
2648 LegalizerHelper::LegalizeResult::UnableToLegalize;
2649}
2650
2651bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2652 MachineInstr &MI, LegalizerHelper &Helper) const {
2653 MachineFunction &MF = *MI.getParent()->getParent();
2654 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2655 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2656
2657 // If stack probing is not enabled for this function, use the default
2658 // lowering.
2659 if (!MF.getFunction().hasFnAttribute(Kind: "probe-stack") ||
2660 MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() !=
2661 "inline-asm") {
2662 Helper.lowerDynStackAlloc(MI);
2663 return true;
2664 }
2665
2666 Register Dst = MI.getOperand(i: 0).getReg();
2667 Register AllocSize = MI.getOperand(i: 1).getReg();
2668 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
2669
2670 assert(MRI.getType(Dst) == LLT::pointer(0, 64) &&
2671 "Unexpected type for dynamic alloca");
2672 assert(MRI.getType(AllocSize) == LLT::scalar(64) &&
2673 "Unexpected type for dynamic alloca");
2674
2675 LLT PtrTy = MRI.getType(Reg: Dst);
2676 Register SPReg =
2677 Helper.getTargetLowering().getStackPointerRegisterToSaveRestore();
2678 Register SPTmp =
2679 Helper.getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
2680 auto NewMI =
2681 MIRBuilder.buildInstr(Opc: AArch64::PROBED_STACKALLOC_DYN, DstOps: {}, SrcOps: {SPTmp});
2682 MRI.setRegClass(Reg: NewMI.getReg(Idx: 0), RC: &AArch64::GPR64commonRegClass);
2683 MIRBuilder.setInsertPt(MBB&: *NewMI->getParent(), II: NewMI);
2684 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
2685
2686 MI.eraseFromParent();
2687 return true;
2688}
2689
2690bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
2691 LegalizerHelper &Helper) const {
2692 MachineIRBuilder &MIB = Helper.MIRBuilder;
2693 auto &AddrVal = MI.getOperand(i: 0);
2694
2695 int64_t IsWrite = MI.getOperand(i: 1).getImm();
2696 int64_t Locality = MI.getOperand(i: 2).getImm();
2697 int64_t IsData = MI.getOperand(i: 3).getImm();
2698
2699 bool IsStream = Locality == 0;
2700 if (Locality != 0) {
2701 assert(Locality <= 3 && "Prefetch locality out-of-range");
2702 // The locality degree is the opposite of the cache speed.
2703 // Put the number the other way around.
2704 // The encoding starts at 0 for level 1
2705 Locality = 3 - Locality;
2706 }
2707
2708 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2709
2710 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
2711 MI.eraseFromParent();
2712 return true;
2713}
2714
2715bool AArch64LegalizerInfo::legalizeConcatVectors(
2716 MachineInstr &MI, MachineRegisterInfo &MRI,
2717 MachineIRBuilder &MIRBuilder) const {
2718 // Widen sub-byte element vectors to byte-sized elements before concatenating.
2719 // This is analogous to SDAG's integer type promotion for sub-byte types.
2720 auto &Concat = cast<GConcatVectors>(Val&: MI);
2721 Register DstReg = Concat.getReg(Idx: 0);
2722 LLT DstTy = MRI.getType(Reg: DstReg);
2723 assert(DstTy.getScalarSizeInBits() < 8 && "Expected dst ty to be < 8b");
2724
2725 unsigned WideEltSize =
2726 std::max(a: 8u, b: (unsigned)PowerOf2Ceil(A: DstTy.getScalarSizeInBits()));
2727 LLT SrcTy = MRI.getType(Reg: Concat.getSourceReg(I: 0));
2728 LLT WideSrcTy = SrcTy.changeElementSize(NewEltSize: WideEltSize);
2729 LLT WideDstTy = DstTy.changeElementSize(NewEltSize: WideEltSize);
2730
2731 SmallVector<Register> WideSrcs;
2732 for (unsigned I = 0; I < Concat.getNumSources(); ++I) {
2733 auto Wide = MIRBuilder.buildAnyExt(Res: WideSrcTy, Op: Concat.getSourceReg(I));
2734 WideSrcs.push_back(Elt: Wide.getReg(Idx: 0));
2735 }
2736
2737 auto WideConcat = MIRBuilder.buildConcatVectors(Res: WideDstTy, Ops: WideSrcs);
2738 MIRBuilder.buildTrunc(Res: DstReg, Op: WideConcat);
2739 MI.eraseFromParent();
2740 return true;
2741}
2742
2743bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
2744 MachineIRBuilder &MIRBuilder,
2745 MachineRegisterInfo &MRI) const {
2746 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
2747
2748 // This function legalizes f64 -> bf16 and f64 -> f16 truncations via f64 ->
2749 // f32 G_FPTRUNC_ODD and f32 -> [b]f16 G_FPTRUNC, which apparently avoids the
2750 // usual double-rounding issue that could be present from using twin
2751 // G_FPTRUNC.
2752
2753 if (DstTy.isBFloat16() && SrcTy.isFloat64()) {
2754 auto Mid = MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {LLT::float32()},
2755 SrcOps: {Src}, Flags: MI.getFlags());
2756 MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC, DstOps: {Dst}, SrcOps: {Mid}, Flags: MI.getFlags());
2757 MI.eraseFromParent();
2758 return true;
2759 }
2760
2761 assert(SrcTy.isFixedVector() && isPowerOf2_32(SrcTy.getNumElements()) &&
2762 "Expected a power of 2 elements");
2763
2764 // We must mutate types here as FPTrunc may be used on a IEEE floating point
2765 // or a brainfloat.
2766 LLT v2s16 = DstTy.changeElementCount(NumElements: 2);
2767 LLT v4s16 = DstTy.changeElementCount(NumElements: 4);
2768 LLT v2s32 = SrcTy.changeElementCount(NumElements: 2).changeElementSize(NewEltSize: 32);
2769 LLT v4s32 = SrcTy.changeElementCount(NumElements: 4).changeElementSize(NewEltSize: 32);
2770 LLT v2s64 = SrcTy.changeElementCount(NumElements: 2);
2771
2772 SmallVector<Register> RegsToUnmergeTo;
2773 SmallVector<Register> TruncOddDstRegs;
2774 SmallVector<Register> RegsToMerge;
2775
2776 unsigned ElemCount = SrcTy.getNumElements();
2777
2778 // Find the biggest size chunks we can work with
2779 int StepSize = ElemCount % 4 ? 2 : 4;
2780
2781 // If we have a power of 2 greater than 2, we need to first unmerge into
2782 // enough pieces
2783 if (ElemCount <= 2)
2784 RegsToUnmergeTo.push_back(Elt: Src);
2785 else {
2786 for (unsigned i = 0; i < ElemCount / 2; ++i)
2787 RegsToUnmergeTo.push_back(Elt: MRI.createGenericVirtualRegister(Ty: v2s64));
2788
2789 MIRBuilder.buildUnmerge(Res: RegsToUnmergeTo, Op: Src);
2790 }
2791
2792 // Create all of the round-to-odd instructions and store them
2793 for (auto SrcReg : RegsToUnmergeTo) {
2794 Register Mid = MIRBuilder
2795 .buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {v2s32}, SrcOps: {SrcReg},
2796 Flags: MI.getFlags())
2797 .getReg(Idx: 0);
2798 TruncOddDstRegs.push_back(Elt: Mid);
2799 }
2800
2801 // Truncate 4s32 to 4s16 if we can to reduce instruction count, otherwise
2802 // truncate 2s32 to 2s16.
2803 unsigned Index = 0;
2804 for (unsigned LoopIter = 0; LoopIter < ElemCount / StepSize; ++LoopIter) {
2805 if (StepSize == 4) {
2806 Register ConcatDst =
2807 MIRBuilder
2808 .buildMergeLikeInstr(
2809 Res: {v4s32}, Ops: {TruncOddDstRegs[Index++], TruncOddDstRegs[Index++]})
2810 .getReg(Idx: 0);
2811
2812 RegsToMerge.push_back(
2813 Elt: MIRBuilder.buildFPTrunc(Res: v4s16, Op: ConcatDst, Flags: MI.getFlags()).getReg(Idx: 0));
2814 } else {
2815 RegsToMerge.push_back(
2816 Elt: MIRBuilder
2817 .buildFPTrunc(Res: v2s16, Op: TruncOddDstRegs[Index++], Flags: MI.getFlags())
2818 .getReg(Idx: 0));
2819 }
2820 }
2821
2822 // If there is only one register, replace the destination
2823 if (RegsToMerge.size() == 1) {
2824 MRI.replaceRegWith(FromReg: Dst, ToReg: RegsToMerge.pop_back_val());
2825 MI.eraseFromParent();
2826 return true;
2827 }
2828
2829 // Merge the rest of the instructions & replace the register
2830 Register Fin = MIRBuilder.buildMergeLikeInstr(Res: DstTy, Ops: RegsToMerge).getReg(Idx: 0);
2831 MRI.replaceRegWith(FromReg: Dst, ToReg: Fin);
2832 MI.eraseFromParent();
2833 return true;
2834}
2835
2836bool AArch64LegalizerInfo::legalizeGetRounding(MachineInstr &MI,
2837 MachineIRBuilder &MIRBuilder,
2838 MachineRegisterInfo &MRI,
2839 LegalizerHelper &Helper) const {
2840 const LLT I32 = LLT::integer(SizeInBits: 32);
2841 const LLT I64 = LLT::integer(SizeInBits: 64);
2842
2843 Register Dst = MI.getOperand(i: 0).getReg();
2844 Register FPCR64 = MRI.createGenericVirtualRegister(Ty: I64);
2845 MachineInstrBuilder GetFPCR =
2846 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_get_fpcr, Res: ArrayRef{FPCR64});
2847
2848 // AArch64 rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3,
2849 // 3->0, so we add one to the FPCR bits for the rounding mode.
2850 // Instead of shifting and then adding as `((FPCR >> 22) + 1) & 0b11` which
2851 // generates 3 instructions, we increment the rounding mode with
2852 // `(FPCR + (1 << 22))` and extract the bits. The shift and addition is done
2853 // in one instruction as `add .., .., #1024, lsl #12`, so overall we generate
2854 // one less instruction.
2855 auto FPCR32 = MIRBuilder.buildTrunc(Res: I32, Op: GetFPCR);
2856 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1U << 22);
2857 auto Added = MIRBuilder.buildAdd(Dst: I32, Src0: FPCR32, Src1: One);
2858 auto LSB = MIRBuilder.buildConstant(Res: I32, Val: 22);
2859 auto Width = MIRBuilder.buildConstant(Res: I32, Val: 2);
2860 MIRBuilder.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {Added, LSB, Width});
2861
2862 MI.eraseFromParent();
2863 return true;
2864}
2865
2866bool AArch64LegalizerInfo::legalizeSetRounding(MachineInstr &MI,
2867 MachineIRBuilder &MIRBuilder,
2868 MachineRegisterInfo &MRI,
2869 LegalizerHelper &Helper) const {
2870 const LLT I32 = LLT::integer(SizeInBits: 32);
2871 const LLT I64 = LLT::integer(SizeInBits: 64);
2872
2873 // AArch64 rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3,
2874 // 3->0, so calculate the new value of FPCR[23:22] as `((arg - 1) & 3) << 22`.
2875 Register RM = MI.getOperand(i: 0).getReg();
2876 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
2877 auto Subtracted = MIRBuilder.buildSub(Dst: I32, Src0: RM, Src1: One);
2878 auto Mask = MIRBuilder.buildConstant(Res: I32, Val: 0b11);
2879 auto Masked = MIRBuilder.buildAnd(Dst: I32, Src0: Subtracted, Src1: Mask);
2880 auto ShiftAmount = MIRBuilder.buildConstant(Res: I32, Val: 22);
2881 auto Shifted = MIRBuilder.buildShl(Dst: I32, Src0: Masked, Src1: ShiftAmount);
2882
2883 // Get current value of FPCR.
2884 MachineInstrBuilder GetFPCR =
2885 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_get_fpcr, Res: {I64});
2886
2887 // (FPCR & ~Mask) | Shifted
2888 auto FPCRMask = MIRBuilder.buildConstant(Res: I64, Val: ~((int64_t)0b11 << 22));
2889 auto FPCRMasked = MIRBuilder.buildAnd(Dst: I64, Src0: GetFPCR, Src1: FPCRMask);
2890 auto ShiftedS64 = MIRBuilder.buildZExt(Res: I64, Op: Shifted);
2891 auto FPCRUpdated = MIRBuilder.buildOr(Dst: I64, Src0: FPCRMasked, Src1: ShiftedS64);
2892
2893 // Write new FPCR.
2894 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_set_fpcr, Res: ArrayRef<Register>())
2895 .addUse(RegNo: FPCRUpdated.getReg(Idx: 0));
2896
2897 MI.eraseFromParent();
2898
2899 return true;
2900}
2901