1//===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64LegalizerInfo.h"
15#include "AArch64Subtarget.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetOpcodes.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/IntrinsicsAArch64.h"
30#include "llvm/IR/Type.h"
31#include "llvm/Support/MathExtras.h"
32#include <initializer_list>
33
34#define DEBUG_TYPE "aarch64-legalinfo"
35
36using namespace llvm;
37using namespace LegalizeActions;
38using namespace LegalizeMutations;
39using namespace LegalityPredicates;
40using namespace MIPatternMatch;
41
42AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
43 : ST(&ST) {
44 using namespace TargetOpcode;
45 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
46 const LLT s8 = LLT::scalar(SizeInBits: 8);
47 const LLT s16 = LLT::scalar(SizeInBits: 16);
48 const LLT s32 = LLT::scalar(SizeInBits: 32);
49 const LLT s64 = LLT::scalar(SizeInBits: 64);
50 const LLT s128 = LLT::scalar(SizeInBits: 128);
51 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
52 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
53 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
54 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
55 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
56 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
57 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
58 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
59 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
60 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
61 const LLT v2p0 = LLT::fixed_vector(NumElements: 2, ScalarTy: p0);
62
63 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
64 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
65 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
66 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
67
68 const LLT bf16 = LLT::bfloat16();
69 const LLT v4bf16 = LLT::fixed_vector(NumElements: 4, ScalarTy: bf16);
70 const LLT v8bf16 = LLT::fixed_vector(NumElements: 8, ScalarTy: bf16);
71
72 const LLT f16 = LLT::float16();
73 const LLT v4f16 = LLT::fixed_vector(NumElements: 4, ScalarTy: f16);
74 const LLT v8f16 = LLT::fixed_vector(NumElements: 8, ScalarTy: f16);
75
76 const LLT f32 = LLT::float32();
77 const LLT v2f32 = LLT::fixed_vector(NumElements: 2, ScalarTy: f32);
78 const LLT v4f32 = LLT::fixed_vector(NumElements: 4, ScalarTy: f32);
79
80 const LLT f64 = LLT::float64();
81 const LLT v2f64 = LLT::fixed_vector(NumElements: 2, ScalarTy: f64);
82
83 const LLT f128 = LLT::float128();
84
85 const LLT i8 = LLT::integer(SizeInBits: 8);
86 const LLT v8i8 = LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
87 const LLT v16i8 = LLT::fixed_vector(NumElements: 16, ScalarTy: i8);
88
89 const LLT i16 = LLT::integer(SizeInBits: 16);
90 const LLT v8i16 = LLT::fixed_vector(NumElements: 8, ScalarTy: i16);
91 const LLT v4i16 = LLT::fixed_vector(NumElements: 4, ScalarTy: i16);
92
93 const LLT i32 = LLT::integer(SizeInBits: 32);
94 const LLT v2i32 = LLT::fixed_vector(NumElements: 2, ScalarTy: i32);
95 const LLT v4i32 = LLT::fixed_vector(NumElements: 4, ScalarTy: i32);
96
97 const LLT i64 = LLT::integer(SizeInBits: 64);
98 const LLT v2i64 = LLT::fixed_vector(NumElements: 2, ScalarTy: i64);
99
100 const LLT i128 = LLT::integer(SizeInBits: 128);
101
102 const LLT nxv16i8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: i8);
103 const LLT nxv8i16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: i16);
104 const LLT nxv4i32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: i32);
105 const LLT nxv2i64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: i64);
106
107 std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
108 v16s8, v8s16, v4s32,
109 v2s64, v2p0,
110 /* End 128bit types */
111 /* Begin 64bit types */
112 v8s8, v4s16, v2s32};
113 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
114 SmallVector<LLT, 8> PackedVectorAllTypesVec(PackedVectorAllTypeList);
115 SmallVector<LLT, 8> ScalarAndPtrTypesVec(ScalarAndPtrTypesList);
116
117 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
118
119 // FIXME: support subtargets which have neon/fp-armv8 disabled.
120 if (!ST.hasNEON() || !ST.hasFPARMv8())
121 return;
122
123 // Some instructions only support s16 if the subtarget has full 16-bit FP
124 // support.
125 const bool HasFP16 = ST.hasFullFP16();
126 const bool HasCSSC = ST.hasCSSC();
127 const bool HasRCPC3 = ST.hasRCPC3();
128 const bool HasSVE = ST.hasSVE();
129
130 getActionDefinitionsBuilder(
131 Opcodes: {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
132 .legalFor(Types: {p0, s8, s16, s32, s64, s128})
133 .legalFor(Types: {v2s8, v4s8, v8s8, v16s8, v2s16, v4s16, v8s16, v2s32, v4s32,
134 v2s64, v2p0})
135 .widenScalarToNextPow2(TypeIdx: 0)
136 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
137 .moreElementsToNextPow2(TypeIdx: 0)
138 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
139 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
140 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
141 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
142 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
143 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
144 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
145
146 getActionDefinitionsBuilder(Opcode: G_PHI)
147 .legalFor(Types: {p0, s16, s32, s64})
148 .legalFor(Types: PackedVectorAllTypeList)
149 .widenScalarToNextPow2(TypeIdx: 0)
150 .moreElementsToNextPow2(TypeIdx: 0)
151 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
152 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: s64)
153 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
154 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
155 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
156 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
157 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
158 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8);
159
160 getActionDefinitionsBuilder(Opcode: G_INSERT)
161 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64, p0}), P1: typeInSet(TypeIdx: 1, TypesInit: {s8, s16, s32}),
162 args: smallerThan(TypeIdx0: 1, TypeIdx1: 0)))
163 .widenScalarToNextPow2(TypeIdx: 0)
164 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
165 .widenScalarToNextPow2(TypeIdx: 1)
166 .minScalar(TypeIdx: 1, Ty: s8)
167 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s32}), TypeIdx: 1, Ty: s16)
168 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s64, p0}), TypeIdx: 1, Ty: s32);
169
170 getActionDefinitionsBuilder(Opcode: G_EXTRACT)
171 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64, p0}),
172 P1: typeInSet(TypeIdx: 1, TypesInit: {s32, s64, s128, p0}), args: smallerThan(TypeIdx0: 0, TypeIdx1: 1)))
173 .widenScalarToNextPow2(TypeIdx: 1)
174 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s128)
175 .widenScalarToNextPow2(TypeIdx: 0)
176 .minScalar(TypeIdx: 0, Ty: s16)
177 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s32}), TypeIdx: 0, Ty: s16)
178 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s64, p0}), TypeIdx: 0, Ty: s32)
179 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s128}), TypeIdx: 0, Ty: s64);
180
181 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB, G_AND, G_OR, G_XOR})
182 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
183 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
184 .widenScalarToNextPow2(TypeIdx: 0)
185 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
186 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
187 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
188 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
189 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
190 .minScalarOrEltIf(
191 Predicate: [=](const LegalityQuery &Query) {
192 return Query.Types[0].getNumElements() <= 2;
193 },
194 TypeIdx: 0, Ty: s32)
195 .minScalarOrEltIf(
196 Predicate: [=](const LegalityQuery &Query) {
197 return Query.Types[0].getNumElements() <= 4;
198 },
199 TypeIdx: 0, Ty: s16)
200 .minScalarOrEltIf(
201 Predicate: [=](const LegalityQuery &Query) {
202 return Query.Types[0].getNumElements() <= 16;
203 },
204 TypeIdx: 0, Ty: s8)
205 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
206 .moreElementsToNextPow2(TypeIdx: 0);
207
208 getActionDefinitionsBuilder(Opcode: G_MUL)
209 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
210 .widenScalarToNextPow2(TypeIdx: 0)
211 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
212 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
213 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
214 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
215 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
216 .minScalarOrEltIf(
217 Predicate: [=](const LegalityQuery &Query) {
218 return Query.Types[0].getNumElements() <= 2;
219 },
220 TypeIdx: 0, Ty: s32)
221 .minScalarOrEltIf(
222 Predicate: [=](const LegalityQuery &Query) {
223 return Query.Types[0].getNumElements() <= 4;
224 },
225 TypeIdx: 0, Ty: s16)
226 .minScalarOrEltIf(
227 Predicate: [=](const LegalityQuery &Query) {
228 return Query.Types[0].getNumElements() <= 16;
229 },
230 TypeIdx: 0, Ty: s8)
231 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
232 .moreElementsToNextPow2(TypeIdx: 0);
233
234 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
235 .customIf(Predicate: [=](const LegalityQuery &Query) {
236 const auto &SrcTy = Query.Types[0];
237 const auto &AmtTy = Query.Types[1];
238 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
239 AmtTy.getSizeInBits() == 32;
240 })
241 .legalFor(Types: {
242 {i32, i32},
243 {i32, i64},
244 {i64, i64},
245 {v8i8, v8i8},
246 {v16i8, v16i8},
247 {v4i16, v4i16},
248 {v8i16, v8i16},
249 {v2i32, v2i32},
250 {v4i32, v4i32},
251 {v2i64, v2i64},
252 })
253 .widenScalarToNextPow2(TypeIdx: 1)
254 .widenScalarToNextPow2(TypeIdx: 0)
255 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
256 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
257 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
258 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
259 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
260 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
261 .moreElementsToNextPow2(TypeIdx: 0)
262 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0)
263 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
264 .minScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, LargeTypeIdx: 0)
265 .maxScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, SmallTypeIdx: 0);
266
267 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
268 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
269 .clampScalarOrElt(TypeIdx: 1, MinTy: s64, MaxTy: s64)
270 .clampNumElements(TypeIdx: 0, MinTy: v2p0, MaxTy: v2p0);
271
272 getActionDefinitionsBuilder(Opcode: G_PTRMASK).legalFor(Types: {{p0, s64}});
273
274 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV})
275 .legalFor(Types: {i32, i64})
276 .libcallFor(Types: {i128})
277 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
278 .widenScalarToNextPow2(TypeIdx: 0)
279 .scalarize(TypeIdx: 0);
280
281 getActionDefinitionsBuilder(Opcodes: {G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
282 .lowerFor(Types: {i8, i16, i32, i64, v2i32, v4i32, v2i64})
283 .libcallFor(Types: {i128})
284 .widenScalarOrEltToNextPow2(TypeIdx: 0)
285 .minScalarOrElt(TypeIdx: 0, Ty: s32)
286 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
287 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
288 .scalarize(TypeIdx: 0);
289
290 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
291 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
292 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
293 .lower();
294
295 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
296 .legalFor(Types: {i64, v16i8, v8i16, v4i32})
297 .lower();
298
299 getActionDefinitionsBuilder(
300 Opcodes: {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT})
301 .lower();
302
303 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX})
304 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
305 .legalFor(Pred: HasCSSC, Types: {i32, i64})
306 .minScalar(Pred: HasCSSC, TypeIdx: 0, Ty: s32)
307 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
308 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
309 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
310 .lower();
311
312 // FIXME: Legal vector types are only legal with NEON.
313 getActionDefinitionsBuilder(Opcode: G_ABS)
314 .legalFor(Pred: HasCSSC, Types: {i32, i64})
315 .legalFor(Types: {v16i8, v8i16, v4i32, v2i64, v2p0, v8i8, v4i16, v2i32})
316 .customIf(Predicate: [=](const LegalityQuery &Q) {
317 // TODO: Fix suboptimal codegen for 128+ bit types.
318 LLT SrcTy = Q.Types[0];
319 return SrcTy.isScalar() && SrcTy.getSizeInBits() < 128;
320 })
321 .widenScalarIf(
322 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v4s8; },
323 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v4i16); })
324 .widenScalarIf(
325 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v2s16; },
326 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v2i32); })
327 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
328 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
329 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
330 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
331 .moreElementsToNextPow2(TypeIdx: 0)
332 .lower();
333
334 getActionDefinitionsBuilder(
335 Opcodes: {G_ABDS, G_ABDU, G_UAVGFLOOR, G_UAVGCEIL, G_SAVGFLOOR, G_SAVGCEIL})
336 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
337 .lower();
338
339 getActionDefinitionsBuilder(
340 Opcodes: {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
341 .legalFor(Types: {{i32, i32}, {i64, i32}})
342 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
343 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
344 .widenScalarToNextPow2(TypeIdx: 0)
345 .lower();
346
347 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR})
348 .customFor(Types: {{i32, i32}, {i32, i64}, {i64, i64}})
349 .lower();
350
351 getActionDefinitionsBuilder(Opcode: G_ROTR)
352 .legalFor(Types: {{i32, i64}, {i64, i64}})
353 .customIf(Predicate: [=](const LegalityQuery &Q) {
354 return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
355 })
356 .lower();
357 getActionDefinitionsBuilder(Opcode: G_ROTL).lower();
358
359 getActionDefinitionsBuilder(Opcodes: {G_SBFX, G_UBFX})
360 .customFor(Types: {{s32, s32}, {s64, s64}});
361
362 auto always = [=](const LegalityQuery &Q) { return true; };
363 getActionDefinitionsBuilder(Opcode: G_CTPOP)
364 .legalFor(Pred: HasCSSC, Types: {{i32, i32}, {i64, i64}})
365 .legalFor(Types: {{v8i8, v8i8}, {v16i8, v16i8}})
366 .customFor(Pred: !HasCSSC, Types: {{s32, s32}, {s64, s64}})
367 .customFor(Types: {{s128, s128},
368 {v4s16, v4s16},
369 {v8s16, v8s16},
370 {v2s32, v2s32},
371 {v4s32, v4s32},
372 {v2s64, v2s64}})
373 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s128)
374 .widenScalarToNextPow2(TypeIdx: 0)
375 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
376 .minScalarEltSameAsIf(Predicate: always, TypeIdx: 1, LargeTypeIdx: 0)
377 .maxScalarEltSameAsIf(Predicate: always, TypeIdx: 1, SmallTypeIdx: 0)
378 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
379 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
380 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
381 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
382 .moreElementsToNextPow2(TypeIdx: 0)
383 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
384
385 getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTLS})
386 .legalFor(Types: {{i32, i32},
387 {i64, i64},
388 {v8i8, v8i8},
389 {v16i8, v16i8},
390 {v4i16, v4i16},
391 {v8i16, v8i16},
392 {v2i32, v2i32},
393 {v4i32, v4i32}})
394 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
395 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
396 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, /*Min=*/MinSize: 8)
397 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
398 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
399 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
400 .moreElementsToNextPow2(TypeIdx: 0)
401 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 32), TypeIdx: 0)
402 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
403
404 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR).lower();
405
406 getActionDefinitionsBuilder(Opcode: G_CTLZ_ZERO_POISON).lower();
407
408 getActionDefinitionsBuilder(Opcode: G_CTTZ)
409 .lowerIf(Predicate: isVector(TypeIdx: 0))
410 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
411 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
412 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1)
413 .legalFor(Pred: HasCSSC, Types: {s32, s64})
414 .customFor(Pred: !HasCSSC, Types: {s32, s64});
415
416 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON).lower();
417
418 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
419 .legalFor(Types: {i32, i64, v8i8, v16i8})
420 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
421 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
422 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
423 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
424 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
425 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
426 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
427 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
428 .moreElementsToNextPow2(TypeIdx: 0)
429 .lower();
430
431 getActionDefinitionsBuilder(Opcode: G_CLMUL).legalFor(Types: {v8i8, v16i8});
432
433 getActionDefinitionsBuilder(Opcode: G_BSWAP)
434 .legalFor(Types: {i32, i64, v4i16, v8i16, v2i32, v4i32, v2i64})
435 .widenScalarOrEltToNextPow2(TypeIdx: 0, MinSize: 16)
436 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
437 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
438 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
439 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
440 .moreElementsToNextPow2(TypeIdx: 0);
441
442 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT})
443 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
444 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
445 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
446 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
447 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
448 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
449 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
450 .moreElementsToNextPow2(TypeIdx: 0)
451 .lower();
452
453 getActionDefinitionsBuilder(
454 Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
455 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
456 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
457 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
458 .legalFor(Pred: HasFP16, Types: {f16, v4f16, v8f16})
459 .libcallFor(Types: {f128})
460 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
461 .widenScalarIf(
462 Predicate: [=](const LegalityQuery &Q) {
463 return (!HasFP16 && Q.Types[0].getScalarType().isFloat16()) ||
464 Q.Types[0].getScalarType().isBFloat16();
465 },
466 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
467 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
468 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
469 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
470 .moreElementsToNextPow2(TypeIdx: 0);
471
472 getActionDefinitionsBuilder(Opcodes: {G_FABS, G_FNEG})
473 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
474 .legalFor(Pred: HasFP16, Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16})
475 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
476 .lowerIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64))
477 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
478 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
479 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
480 .moreElementsToNextPow2(TypeIdx: 0)
481 .lowerFor(Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16});
482
483 getActionDefinitionsBuilder(Opcodes: {G_FREM, G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
484 G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10,
485 G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH,
486 G_FSINH, G_FTANH, G_FMODF})
487 .libcallFor(Types: {f32, f64, f128})
488 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
489 .scalarize(TypeIdx: 0);
490 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
491 .libcallFor(Types: {{f32, i32}, {f64, i32}, {f128, i32}})
492 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
493 .scalarize(TypeIdx: 0);
494
495 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_INTRINSIC_LRINT})
496 .legalFor(Types: {{i32, f32}, {i32, f64}, {i64, f32}, {i64, f64}})
497 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {i64, f16}})
498 .minScalar(TypeIdx: 1, Ty: s32)
499 .libcallFor(Types: {{s64, s128}})
500 .lower();
501 getActionDefinitionsBuilder(Opcodes: {G_LLROUND, G_INTRINSIC_LLRINT})
502 .legalFor(Types: {{i64, f32}, {i64, f64}})
503 .legalFor(Pred: HasFP16, Types: {{i64, f16}})
504 .minScalar(TypeIdx: 0, Ty: s64)
505 .minScalar(TypeIdx: 1, Ty: s32)
506 .libcallFor(Types: {{s64, s128}})
507 .lower();
508
509 // TODO: Custom legalization for mismatched types.
510 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
511 .moreElementsIf(
512 Predicate: [](const LegalityQuery &Query) { return Query.Types[0].isScalar(); },
513 Mutation: [=](const LegalityQuery &Query) {
514 const LLT Ty = Query.Types[0];
515 return std::pair(0, LLT::fixed_vector(NumElements: Ty == s16 ? 4 : 2, ScalarTy: Ty));
516 })
517 .lower();
518
519 getActionDefinitionsBuilder(Opcode: G_FMAD).lower();
520
521 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
522 auto &Actions = getActionDefinitionsBuilder(Opcode: Op);
523
524 if (Op == G_SEXTLOAD)
525 Actions.lowerIf(Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered));
526
527 // Atomics have zero extending behavior.
528 Actions
529 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8},
530 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8},
531 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
532 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 2},
533 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 2},
534 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 4},
535 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
536 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
537 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}})
538 .widenScalarToNextPow2(TypeIdx: 0)
539 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
540 // TODO: We could support sum-of-pow2's but the lowering code doesn't know
541 // how to do that yet.
542 .unsupportedIfMemSizeNotPow2()
543 // Lower anything left over into G_*EXT and G_LOAD
544 .lower();
545 }
546
547 auto IsPtrVecPred = [=](const LegalityQuery &Query) {
548 const LLT &ValTy = Query.Types[0];
549 return ValTy.isPointerVector() && ValTy.getAddressSpace() == 0;
550 };
551
552 getActionDefinitionsBuilder(Opcode: G_LOAD)
553 .customIf(Predicate: [=](const LegalityQuery &Query) {
554 return HasRCPC3 && Query.Types[0] == s128 &&
555 Query.MMODescrs[0].Ordering == AtomicOrdering::Acquire;
556 })
557 .customIf(Predicate: [=](const LegalityQuery &Query) {
558 return Query.Types[0] == s128 &&
559 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
560 })
561 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8},
562 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8},
563 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
564 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
565 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
566 {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8},
567 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8},
568 {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
569 {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8},
570 {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
571 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8},
572 {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8},
573 {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
574 // These extends are also legal
575 .legalForTypesWithMemDesc(
576 TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}})
577 .legalForTypesWithMemDesc(TypesAndMemDesc: {
578 // SVE vscale x 128 bit base sizes
579 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
580 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
581 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
582 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
583 })
584 .widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
585 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
586 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
587 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
588 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
589 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
590 .lowerIfMemSizeNotByteSizePow2()
591 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
592 .narrowScalarIf(
593 Predicate: [=](const LegalityQuery &Query) {
594 // Clamp extending load results to 32-bits.
595 return Query.Types[0].isScalar() &&
596 Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
597 Query.Types[0].getSizeInBits() > 32;
598 },
599 Mutation: changeTo(TypeIdx: 0, Ty: i32))
600 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
601 .bitcastIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
602 Mutation: [=](const LegalityQuery &Query) {
603 const LLT VecTy = Query.Types[0];
604 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
605 })
606 .customIf(Predicate: IsPtrVecPred)
607 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
608 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
609
610 getActionDefinitionsBuilder(Opcode: G_STORE)
611 .customIf(Predicate: [=](const LegalityQuery &Query) {
612 return HasRCPC3 && Query.Types[0] == s128 &&
613 Query.MMODescrs[0].Ordering == AtomicOrdering::Release;
614 })
615 .customIf(Predicate: [=](const LegalityQuery &Query) {
616 return Query.Types[0] == s128 &&
617 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
618 })
619 .widenScalarIf(
620 Predicate: all(P0: scalarNarrowerThan(TypeIdx: 0, Size: 32),
621 P1: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Release)),
622 Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: s32))
623 .legalForTypesWithMemDesc(
624 TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s16
625 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s32
626 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s64
627 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s32
628 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s64
629 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
630 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}, // truncstorei32 from s64
631 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
632 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
633 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
634 .legalForTypesWithMemDesc(TypesAndMemDesc: {
635 // SVE vscale x 128 bit base sizes
636 // TODO: Add nxv2p0. Consider bitcastIf.
637 // See #92130
638 // https://github.com/llvm/llvm-project/pull/92130#discussion_r1616888461
639 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
640 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
641 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
642 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
643 })
644 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
645 .minScalarOrElt(TypeIdx: 0, Ty: s8)
646 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
647 return Query.Types[0].isScalar() &&
648 Query.Types[0] != Query.MMODescrs[0].MemoryTy;
649 })
650 // Maximum: sN * k = 128
651 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
652 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
653 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
654 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
655 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
656 .lowerIfMemSizeNotPow2()
657 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
658 .bitcastIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
659 P1: LegalityPredicate([=](const LegalityQuery &Query) {
660 return Query.Types[0].getSizeInBits() ==
661 Query.MMODescrs[0].MemoryTy.getSizeInBits();
662 })),
663 Mutation: [=](const LegalityQuery &Query) {
664 const LLT VecTy = Query.Types[0];
665 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
666 })
667 .customIf(Predicate: IsPtrVecPred)
668 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
669 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
670 .lower();
671
672 getActionDefinitionsBuilder(Opcode: G_INDEXED_STORE)
673 // Idx 0 == Ptr, Idx 1 == Val
674 // TODO: we can implement legalizations but as of now these are
675 // generated in a very specific way.
676 .legalForTypesWithMemDesc(TypesAndMemDesc: {
677 {.Type0: p0, .Type1: s8, .MemTy: s8, .Align: 8},
678 {.Type0: p0, .Type1: s16, .MemTy: s16, .Align: 8},
679 {.Type0: p0, .Type1: s32, .MemTy: s8, .Align: 8},
680 {.Type0: p0, .Type1: s32, .MemTy: s16, .Align: 8},
681 {.Type0: p0, .Type1: s32, .MemTy: s32, .Align: 8},
682 {.Type0: p0, .Type1: s64, .MemTy: s64, .Align: 8},
683 {.Type0: p0, .Type1: p0, .MemTy: p0, .Align: 8},
684 {.Type0: p0, .Type1: v8s8, .MemTy: v8s8, .Align: 8},
685 {.Type0: p0, .Type1: v16s8, .MemTy: v16s8, .Align: 8},
686 {.Type0: p0, .Type1: v4s16, .MemTy: v4s16, .Align: 8},
687 {.Type0: p0, .Type1: v8s16, .MemTy: v8s16, .Align: 8},
688 {.Type0: p0, .Type1: v2s32, .MemTy: v2s32, .Align: 8},
689 {.Type0: p0, .Type1: v4s32, .MemTy: v4s32, .Align: 8},
690 {.Type0: p0, .Type1: v2s64, .MemTy: v2s64, .Align: 8},
691 {.Type0: p0, .Type1: v2p0, .MemTy: v2p0, .Align: 8},
692 {.Type0: p0, .Type1: s128, .MemTy: s128, .Align: 8},
693 })
694 .unsupported();
695
696 auto IndexedLoadBasicPred = [=](const LegalityQuery &Query) {
697 LLT LdTy = Query.Types[0];
698 LLT PtrTy = Query.Types[1];
699 if (!llvm::is_contained(Range: PackedVectorAllTypesVec, Element: LdTy) &&
700 !llvm::is_contained(Range: ScalarAndPtrTypesVec, Element: LdTy) && LdTy != s128)
701 return false;
702 if (PtrTy != p0)
703 return false;
704 return true;
705 };
706 getActionDefinitionsBuilder(Opcode: G_INDEXED_LOAD)
707 .unsupportedIf(
708 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
709 .legalIf(Predicate: IndexedLoadBasicPred)
710 .unsupported();
711 getActionDefinitionsBuilder(Opcodes: {G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD})
712 .unsupportedIf(
713 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
714 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64}),
715 P1: LegalityPredicate([=](const LegalityQuery &Q) {
716 LLT LdTy = Q.Types[0];
717 LLT PtrTy = Q.Types[1];
718 LLT MemTy = Q.MMODescrs[0].MemoryTy;
719 if (PtrTy != p0)
720 return false;
721 if (LdTy == s16)
722 return MemTy == s8;
723 if (LdTy == s32)
724 return MemTy == s8 || MemTy == s16;
725 if (LdTy == s64)
726 return MemTy == s8 || MemTy == s16 || MemTy == s32;
727 return false;
728 })))
729 .unsupported();
730
731 // Constants
732 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
733 .legalFor(Types: {p0, s8, s16, s32, s64})
734 .widenScalarToNextPow2(TypeIdx: 0)
735 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64);
736 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
737 .legalFor(Types: {s16, s32, s64, s128});
738
739 // FIXME: fix moreElementsToNextPow2
740 getActionDefinitionsBuilder(Opcode: G_ICMP)
741 .legalFor(Types: {{i32, i32}, {i32, i64}, {i32, p0}})
742 .widenScalarOrEltToNextPow2(TypeIdx: 1)
743 .minScalarOrElt(TypeIdx: 1, Ty: s8)
744 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
745 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
746 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
747 .minScalarEltSameAsIf(
748 Predicate: [=](const LegalityQuery &Query) {
749 const LLT &Ty = Query.Types[0];
750 const LLT &SrcTy = Query.Types[1];
751 return Ty.isVector() && !SrcTy.isPointerVector() &&
752 Ty.getElementType() != SrcTy.getElementType();
753 },
754 TypeIdx: 0, LargeTypeIdx: 1)
755 .minScalarOrEltIf(
756 Predicate: [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
757 TypeIdx: 1, Ty: s32)
758 .minScalarOrEltIf(
759 Predicate: [=](const LegalityQuery &Query) {
760 return Query.Types[1].isPointerVector();
761 },
762 TypeIdx: 0, Ty: s64)
763 .moreElementsToNextPow2(TypeIdx: 1)
764 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
765 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
766 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
767 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
768 .clampNumElements(TypeIdx: 1, MinTy: v2p0, MaxTy: v2p0)
769 .customIf(Predicate: isVector(TypeIdx: 0));
770
771 getActionDefinitionsBuilder(Opcode: G_FCMP)
772 .legalFor(Types: {{i32, f32},
773 {i32, f64},
774 {v4i32, v4f32},
775 {v2i32, v2f32},
776 {v2i64, v2f64}})
777 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
778 .widenScalarOrEltToNextPow2(TypeIdx: 1)
779 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
780 .widenScalarIf(
781 Predicate: [=](const LegalityQuery &Q) {
782 return (!HasFP16 && Q.Types[1].getScalarType().isFloat16()) ||
783 Q.Types[1].getScalarType().isBFloat16();
784 },
785 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
786 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
787 .minScalarEltSameAsIf(
788 Predicate: [=](const LegalityQuery &Query) {
789 const LLT &Ty = Query.Types[0];
790 const LLT &SrcTy = Query.Types[1];
791 return Ty.isVector() && !SrcTy.isPointerVector() &&
792 Ty.getElementType() != SrcTy.getElementType();
793 },
794 TypeIdx: 0, LargeTypeIdx: 1)
795 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
796 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
797 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
798 .moreElementsToNextPow2(TypeIdx: 1)
799 .libcallFor(Types: {{s32, s128}});
800
801 // Extensions
802 auto ExtLegalFunc = [=](const LegalityQuery &Query) {
803 unsigned DstSize = Query.Types[0].getSizeInBits();
804
805 // Handle legal vectors using legalFor
806 if (Query.Types[0].isVector())
807 return false;
808
809 if (DstSize < 8 || DstSize >= 128 || !isPowerOf2_32(Value: DstSize))
810 return false; // Extending to a scalar s128 needs narrowing.
811
812 const LLT &SrcTy = Query.Types[1];
813
814 // Make sure we fit in a register otherwise. Don't bother checking that
815 // the source type is below 128 bits. We shouldn't be allowing anything
816 // through which is wider than the destination in the first place.
817 unsigned SrcSize = SrcTy.getSizeInBits();
818 if (SrcSize < 8 || !isPowerOf2_32(Value: SrcSize))
819 return false;
820
821 return true;
822 };
823 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
824 .legalIf(Predicate: ExtLegalFunc)
825 .legalFor(Types: {{v8s16, v8s8}, {v4s32, v4s16}, {v2s64, v2s32}})
826 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64) // Just for s128, others are handled above.
827 .moreElementsToNextPow2(TypeIdx: 0)
828 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
829 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
830 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
831 // Tries to convert a large EXTEND into two smaller EXTENDs
832 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
833 return (Query.Types[0].getScalarSizeInBits() >
834 Query.Types[1].getScalarSizeInBits() * 2) &&
835 Query.Types[0].isVector() &&
836 (Query.Types[1].getScalarSizeInBits() == 8 ||
837 Query.Types[1].getScalarSizeInBits() == 16);
838 })
839 .clampMinNumElements(TypeIdx: 1, EltTy: s8, MinElements: 8)
840 .clampMinNumElements(TypeIdx: 1, EltTy: s16, MinElements: 4)
841 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
842
843 getActionDefinitionsBuilder(Opcode: G_TRUNC)
844 .legalFor(Types: {{v8s8, v8s16}, {v4s16, v4s32}, {v2s32, v2s64}})
845 .moreElementsToNextPow2(TypeIdx: 0)
846 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 8)
847 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 4)
848 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 2)
849 .minScalarOrEltIf(
850 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
851 TypeIdx: 0, Ty: s8)
852 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
853 LLT DstTy = Query.Types[0];
854 LLT SrcTy = Query.Types[1];
855 return DstTy.isVector() && SrcTy.getSizeInBits() > 128 &&
856 DstTy.getScalarSizeInBits() * 2 <= SrcTy.getScalarSizeInBits();
857 })
858 .clampMinNumElements(TypeIdx: 0, EltTy: s8, MinElements: 8)
859 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 4)
860 .alwaysLegal();
861
862 getActionDefinitionsBuilder(Opcodes: {G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
863 .legalFor(Types: {{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
864 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v8s8)
865 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v4s16)
866 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v2s32)
867 .lower();
868
869 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
870 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
871 .maxScalar(TypeIdx: 0, Ty: s64)
872 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
873 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
874 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
875 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
876 .lower();
877
878 // FP conversions
879 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
880 .legalFor(
881 Types: {{f16, f32}, {f16, f64}, {f32, f64}, {v4f16, v4f32}, {v2f32, v2f64}})
882 .legalFor(Pred: ST.hasBF16(), Types: {{bf16, f32}, {v4bf16, v4f32}})
883 .libcallFor(Types: {{f16, f128}, {f32, f128}, {f64, f128}})
884 .moreElementsToNextPow2(TypeIdx: 1)
885 .customIf(Predicate: [](const LegalityQuery &Q) {
886 LLT DstTy = Q.Types[0];
887 LLT SrcTy = Q.Types[1];
888 return SrcTy.getScalarSizeInBits() == 64 &&
889 DstTy.getScalarSizeInBits() == 16;
890 })
891 .lowerFor(Types: {{bf16, f32}, {v4bf16, v4f32}})
892 // Clamp based on input
893 .clampNumElements(TypeIdx: 1, MinTy: v4s32, MaxTy: v4s32)
894 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
895 .scalarize(TypeIdx: 0);
896
897 getActionDefinitionsBuilder(Opcode: G_FPEXT)
898 .legalFor(Types: {{f32, f16},
899 {f64, f16},
900 {f32, bf16},
901 {f64, f32},
902 {v4f32, v4f16},
903 {v4f32, v4bf16},
904 {v2f64, v2f32}})
905 .libcallFor(Types: {{f128, f64}, {f128, f32}, {f128, f16}})
906 .moreElementsToNextPow2(TypeIdx: 0)
907 .widenScalarIf(
908 Predicate: [](const LegalityQuery &Q) {
909 LLT DstTy = Q.Types[0];
910 LLT SrcTy = Q.Types[1];
911 return SrcTy.isVector() && DstTy.isVector() &&
912 SrcTy.getScalarSizeInBits() == 16 &&
913 DstTy.getScalarSizeInBits() == 64;
914 },
915 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
916 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
917 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
918 .scalarize(TypeIdx: 0);
919
920 // Conversions
921 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
922 .legalFor(Types: {{i32, f32},
923 {i64, f32},
924 {i32, f64},
925 {i64, f64},
926 {v2i32, v2f32},
927 {v4i32, v4f32},
928 {v2i64, v2f64}})
929 .legalFor(Pred: HasFP16,
930 Types: {{i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
931 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
932 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
933 // The range of a fp16 value fits into an i17, so we can lower the width
934 // to i64.
935 .narrowScalarIf(
936 Predicate: [=](const LegalityQuery &Query) {
937 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
938 },
939 Mutation: changeTo(TypeIdx: 0, Ty: i64))
940 .moreElementsToNextPow2(TypeIdx: 0)
941 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0)
942 .minScalar(TypeIdx: 0, Ty: s32)
943 .widenScalarIf(
944 Predicate: [HasFP16](const LegalityQuery &Query) {
945 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
946 Query.Types[1].getScalarType().isBFloat16();
947 },
948 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
949 .widenScalarIf(
950 Predicate: [=](const LegalityQuery &Query) {
951 return Query.Types[0].getScalarSizeInBits() <= 64 &&
952 Query.Types[0].getScalarSizeInBits() >
953 Query.Types[1].getScalarSizeInBits();
954 },
955 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
956 .widenScalarIf(
957 Predicate: [=](const LegalityQuery &Query) {
958 return Query.Types[1].getScalarSizeInBits() <= 64 &&
959 Query.Types[0].getScalarSizeInBits() <
960 Query.Types[1].getScalarSizeInBits();
961 },
962 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
963 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
964 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
965 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
966 .libcallFor(
967 Types: {{i32, f128}, {i64, f128}, {i128, f128}, {i128, f32}, {i128, f64}});
968
969 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
970 .legalFor(Types: {{i32, f32},
971 {i64, f32},
972 {i32, f64},
973 {i64, f64},
974 {v2i32, v2f32},
975 {v4i32, v4f32},
976 {v2i64, v2f64}})
977 .legalFor(
978 Pred: HasFP16,
979 Types: {{i16, f16}, {i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
980 // Handle types larger than i64 by scalarizing/lowering.
981 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
982 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
983 // The range of a fp16 value fits into an i17, so we can lower the width
984 // to i64.
985 .narrowScalarIf(
986 Predicate: [=](const LegalityQuery &Query) {
987 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
988 },
989 Mutation: changeTo(TypeIdx: 0, Ty: i64))
990 .lowerIf(Predicate: ::any(P0: scalarWiderThan(TypeIdx: 0, Size: 64), P1: scalarWiderThan(TypeIdx: 1, Size: 64)), Mutation: 0)
991 .moreElementsToNextPow2(TypeIdx: 0)
992 .widenScalarToNextPow2(TypeIdx: 0, /*MinSize=*/32)
993 .minScalar(TypeIdx: 0, Ty: s32)
994 .widenScalarIf(
995 Predicate: [HasFP16](const LegalityQuery &Query) {
996 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
997 Query.Types[1].getScalarType().isBFloat16();
998 },
999 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
1000 .widenScalarIf(
1001 Predicate: [=](const LegalityQuery &Query) {
1002 unsigned ITySize = Query.Types[0].getScalarSizeInBits();
1003 return (ITySize == 16 || ITySize == 32 || ITySize == 64) &&
1004 ITySize > Query.Types[1].getScalarSizeInBits();
1005 },
1006 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1007 .widenScalarIf(
1008 Predicate: [=](const LegalityQuery &Query) {
1009 unsigned FTySize = Query.Types[1].getScalarSizeInBits();
1010 return (FTySize == 16 || FTySize == 32 || FTySize == 64) &&
1011 Query.Types[0].getScalarSizeInBits() < FTySize;
1012 },
1013 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1014 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1015 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1016 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1017 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1018
1019 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
1020 .legalFor(Types: {{f32, i32},
1021 {f64, i32},
1022 {f32, i64},
1023 {f64, i64},
1024 {v2f32, v2i32},
1025 {v4f32, v4i32},
1026 {v2f64, v2i64}})
1027 .legalFor(Pred: HasFP16,
1028 Types: {{f16, i32}, {f16, i64}, {v4f16, v4i16}, {v8f16, v8i16}})
1029 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1030 return Query.Types[0].getScalarType().isBFloat16();
1031 })
1032 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
1033 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1034 .moreElementsToNextPow2(TypeIdx: 1)
1035 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1)
1036 .minScalar(TypeIdx: 1, Ty: f32)
1037 .lowerIf(Predicate: [](const LegalityQuery &Query) {
1038 return Query.Types[1].isVector() &&
1039 Query.Types[1].getScalarSizeInBits() == 64 &&
1040 Query.Types[0].getScalarSizeInBits() == 16;
1041 })
1042 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, /*MinSize=*/HasFP16 ? 16 : 32)
1043 .scalarizeIf(
1044 // v2i64->v2f32 needs to scalarize to avoid double-rounding issues.
1045 Predicate: [](const LegalityQuery &Query) {
1046 return Query.Types[0].getScalarSizeInBits() == 32 &&
1047 Query.Types[1].getScalarSizeInBits() == 64;
1048 },
1049 TypeIdx: 0)
1050 .widenScalarIf(
1051 Predicate: [](const LegalityQuery &Query) {
1052 return Query.Types[1].getScalarSizeInBits() <= 64 &&
1053 Query.Types[0].getScalarSizeInBits() <
1054 Query.Types[1].getScalarSizeInBits();
1055 },
1056 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1057 .widenScalarIf(
1058 Predicate: [](const LegalityQuery &Query) {
1059 return Query.Types[0].getScalarSizeInBits() <= 64 &&
1060 Query.Types[0].getScalarSizeInBits() >
1061 Query.Types[1].getScalarSizeInBits();
1062 },
1063 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1064 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1065 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1066 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1067 .libcallFor(Types: {{f16, i128},
1068 {f32, i128},
1069 {f64, i128},
1070 {f128, i128},
1071 {f128, i32},
1072 {f128, i64}});
1073
1074 // Control-flow
1075 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
1076 getActionDefinitionsBuilder(Opcode: G_BRCOND)
1077 .legalFor(Types: {s32})
1078 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32);
1079 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
1080
1081 getActionDefinitionsBuilder(Opcode: G_SELECT)
1082 .legalFor(Types: {{s32, s32}, {s64, s32}, {p0, s32}})
1083 .widenScalarToNextPow2(TypeIdx: 0)
1084 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
1085 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s32)
1086 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1087 .minScalarEltSameAsIf(Predicate: all(P0: isVector(TypeIdx: 0), P1: isVector(TypeIdx: 1)), TypeIdx: 1, LargeTypeIdx: 0)
1088 .lowerIf(Predicate: isVector(TypeIdx: 0));
1089
1090 // Pointer-handling
1091 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
1092
1093 if (TM.getCodeModel() == CodeModel::Small)
1094 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).custom();
1095 else
1096 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: {p0});
1097
1098 getActionDefinitionsBuilder(Opcode: G_PTRAUTH_GLOBAL_VALUE)
1099 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: p0), P1: typeIs(TypeIdx: 1, TypesInit: p0)));
1100
1101 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
1102 .legalFor(Types: {{i64, p0}, {v2i64, v2p0}})
1103 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 64)
1104 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64)
1105 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1106
1107 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
1108 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1109 return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
1110 })
1111 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
1112 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2);
1113
1114 // Casts for 32 and 64-bit width type are just copies.
1115 // Same for 128-bit width type, except they are on the FPR bank.
1116 getActionDefinitionsBuilder(Opcode: G_BITCAST)
1117 .legalForCartesianProduct(Types: {s16})
1118 // Keeping 32-bit instructions legal to prevent regression in some tests
1119 .legalForCartesianProduct(Types: {s32, v2s16, v4s8})
1120 .legalForCartesianProduct(Types: {s64, v8s8, v4s16, v2s32})
1121 .legalForCartesianProduct(Types: {s128, v16s8, v8s16, v4s32, v2s64, v2p0})
1122 .customIf(Predicate: [=](const LegalityQuery &Query) {
1123 // Handle casts from i1 vectors to scalars.
1124 LLT DstTy = Query.Types[0];
1125 LLT SrcTy = Query.Types[1];
1126 return DstTy.isScalar() && SrcTy.isVector() &&
1127 SrcTy.getScalarSizeInBits() == 1;
1128 })
1129 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
1130 return Query.Types[0].isVector() != Query.Types[1].isVector();
1131 })
1132 // moreElementsToNextPow2 cannot pad the source to match, so lower
1133 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
1134 LLT DstTy = Query.Types[0];
1135 LLT SrcTy = Query.Types[1];
1136 if (!DstTy.isFixedVector() || !SrcTy.isFixedVector())
1137 return false;
1138 unsigned MoreElts = 1u << Log2_32_Ceil(Value: DstTy.getNumElements());
1139 return SrcTy.getNumElements() * MoreElts % DstTy.getNumElements() != 0;
1140 })
1141 .moreElementsToNextPow2(TypeIdx: 0)
1142 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1143 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1144 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1145 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1146 .lower();
1147
1148 getActionDefinitionsBuilder(Opcode: G_VASTART).legalFor(Types: {p0});
1149
1150 // va_list must be a pointer, but most sized types are pretty easy to handle
1151 // as the destination.
1152 getActionDefinitionsBuilder(Opcode: G_VAARG)
1153 .customForCartesianProduct(Types0: {s8, s16, s32, s64, p0}, Types1: {p0})
1154 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
1155 .widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8);
1156
1157 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
1158 .lowerIf(
1159 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64, s128}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
1160
1161 bool UseOutlineAtomics = ST.outlineAtomics() && !ST.hasLSE();
1162
1163 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG)
1164 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1165 .customFor(Pred: !UseOutlineAtomics, Types: {{s128, p0}})
1166 .libcallFor(Pred: UseOutlineAtomics,
1167 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {s128, p0}})
1168 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1169
1170 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD,
1171 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
1172 G_ATOMICRMW_XOR})
1173 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1174 .libcallFor(Pred: UseOutlineAtomics,
1175 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
1176 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1177
1178 // Do not outline these atomics operations, as per comment in
1179 // AArch64ISelLowering.cpp's shouldExpandAtomicRMWInIR().
1180 getActionDefinitionsBuilder(
1181 Opcodes: {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
1182 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64}), P1: typeIs(TypeIdx: 1, TypesInit: p0)))
1183 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1184
1185 getActionDefinitionsBuilder(Opcode: G_BLOCK_ADDR).legalFor(Types: {p0});
1186
1187 // Merge/Unmerge
1188 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
1189 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
1190 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
1191 getActionDefinitionsBuilder(Opcode: Op)
1192 .widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: 8)
1193 // Above s64 lowered shifts narrow back to a merge and never terminate
1194 .lowerIf(Predicate: [=](const LegalityQuery &Q) {
1195 const LLT BigTy = Q.Types[BigTyIdx];
1196 return BigTy.isScalar() && !isPowerOf2_32(Value: BigTy.getSizeInBits()) &&
1197 BigTy.getSizeInBits() < 64;
1198 })
1199 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: 32)
1200 .clampScalar(TypeIdx: LitTyIdx, MinTy: s8, MaxTy: s64)
1201 .clampScalar(TypeIdx: BigTyIdx, MinTy: s32, MaxTy: s128)
1202 .legalIf(Predicate: [=](const LegalityQuery &Q) {
1203 switch (Q.Types[BigTyIdx].getSizeInBits()) {
1204 case 32:
1205 case 64:
1206 case 128:
1207 break;
1208 default:
1209 return false;
1210 }
1211 switch (Q.Types[LitTyIdx].getSizeInBits()) {
1212 case 8:
1213 case 16:
1214 case 32:
1215 case 64:
1216 return true;
1217 default:
1218 return false;
1219 }
1220 });
1221 }
1222
1223 // TODO : nxv4s16, nxv2s16, nxv2s32
1224 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
1225 .legalFor(Pred: HasSVE, Types: {{s16, nxv16s8, s64},
1226 {s16, nxv8s16, s64},
1227 {s32, nxv4s32, s64},
1228 {s64, nxv2s64, s64}})
1229 .unsupportedIf(Predicate: [=](const LegalityQuery &Query) {
1230 const LLT &EltTy = Query.Types[1].getElementType();
1231 if (Query.Types[1].isScalableVector())
1232 return false;
1233 return Query.Types[0] != EltTy;
1234 })
1235 .minScalar(TypeIdx: 2, Ty: s64)
1236 .customIf(Predicate: [=](const LegalityQuery &Query) {
1237 const LLT &VecTy = Query.Types[1];
1238 return VecTy == v8s8 || VecTy == v16s8 || VecTy == v2s16 ||
1239 VecTy == v4s16 || VecTy == v8s16 || VecTy == v2s32 ||
1240 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2p0;
1241 })
1242 .minScalarOrEltIf(
1243 Predicate: [=](const LegalityQuery &Query) {
1244 // We want to promote to <M x s1> to <M x s64> if that wouldn't
1245 // cause the total vec size to be > 128b.
1246 return Query.Types[1].isFixedVector() &&
1247 Query.Types[1].getNumElements() <= 2;
1248 },
1249 TypeIdx: 0, Ty: s64)
1250 .minScalarOrEltIf(
1251 Predicate: [=](const LegalityQuery &Query) {
1252 return Query.Types[1].isFixedVector() &&
1253 Query.Types[1].getNumElements() <= 4;
1254 },
1255 TypeIdx: 0, Ty: s32)
1256 .minScalarOrEltIf(
1257 Predicate: [=](const LegalityQuery &Query) {
1258 return Query.Types[1].isFixedVector() &&
1259 Query.Types[1].getNumElements() <= 8;
1260 },
1261 TypeIdx: 0, Ty: s16)
1262 .minScalarOrEltIf(
1263 Predicate: [=](const LegalityQuery &Query) {
1264 return Query.Types[1].isFixedVector() &&
1265 Query.Types[1].getNumElements() <= 16;
1266 },
1267 TypeIdx: 0, Ty: s8)
1268 .minScalarOrElt(TypeIdx: 0, Ty: s8) // Worst case, we need at least s8.
1269 .moreElementsToNextPow2(TypeIdx: 1)
1270 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1271 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1272 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1273 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1274 .clampMaxNumElements(TypeIdx: 1, EltTy: p0, MaxElements: 2)
1275 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1);
1276
1277 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
1278 .legalIf(
1279 Predicate: typeInSet(TypeIdx: 0, TypesInit: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64, v2p0}))
1280 .legalFor(Pred: HasSVE, Types: {{nxv16s8, s32, s64},
1281 {nxv8s16, s32, s64},
1282 {nxv4s32, s32, s64},
1283 {nxv2s64, s64, s64}})
1284 .moreElementsToNextPow2(TypeIdx: 0)
1285 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1286 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1287 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1288 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1289 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1290 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
1291 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
1292
1293 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
1294 .legalFor(Types: {{v8s8, s8},
1295 {v16s8, s8},
1296 {v4s16, s16},
1297 {v8s16, s16},
1298 {v2s32, s32},
1299 {v4s32, s32},
1300 {v2s64, s64},
1301 {v2p0, p0}})
1302 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1303 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1304 .minScalarOrElt(TypeIdx: 0, Ty: s8)
1305 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1306 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1307 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0);
1308
1309 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR_TRUNC).lower();
1310
1311 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
1312 .legalFor(Types: {{v8s8, s8},
1313 {v8s8, s32},
1314 {v16s8, s8},
1315 {v16s8, s32},
1316 {v4s16, s16},
1317 {v4s16, s32},
1318 {v8s16, s16},
1319 {v8s16, s32},
1320 {v2s32, s32},
1321 {v4s32, s32},
1322 {v2s64, s64},
1323 {v2p0, p0}});
1324
1325 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
1326 .legalIf(Predicate: [=](const LegalityQuery &Query) {
1327 const LLT &DstTy = Query.Types[0];
1328 const LLT &SrcTy = Query.Types[1];
1329 // For now just support the TBL2 variant which needs the source vectors
1330 // to be the same size as the dest.
1331 if (DstTy != SrcTy)
1332 return false;
1333 return llvm::is_contained(
1334 Set: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64}, Element: DstTy);
1335 })
1336 .moreElementsIf(
1337 Predicate: [](const LegalityQuery &Query) {
1338 return Query.Types[0].getNumElements() >
1339 Query.Types[1].getNumElements();
1340 },
1341 Mutation: changeTo(TypeIdx: 1, FromTypeIdx: 0))
1342 .moreElementsToNextPow2(TypeIdx: 0)
1343 .moreElementsIf(
1344 Predicate: [](const LegalityQuery &Query) {
1345 return Query.Types[0].getNumElements() <
1346 Query.Types[1].getNumElements();
1347 },
1348 Mutation: changeTo(TypeIdx: 0, FromTypeIdx: 1))
1349 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
1350 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1351 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1352 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1353 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1354 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1355 .bitcastIf(Predicate: isPointerVector(TypeIdx: 0), Mutation: [=](const LegalityQuery &Query) {
1356 // Bitcast pointers vector to i64.
1357 const LLT DstTy = Query.Types[0];
1358 return std::pair(
1359 0, LLT::vector(EC: DstTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 64)));
1360 });
1361
1362 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
1363 .legalFor(Types: {{v16s8, v8s8}, {v8s16, v4s16}, {v4s32, v2s32}})
1364 .customIf(Predicate: [=](const LegalityQuery &Query) {
1365 return Query.Types[0].isFixedVector() &&
1366 Query.Types[0].getScalarSizeInBits() < 8;
1367 })
1368 .bitcastIf(
1369 Predicate: [=](const LegalityQuery &Query) {
1370 return Query.Types[0].isFixedVector() &&
1371 Query.Types[1].isFixedVector() &&
1372 Query.Types[0].getScalarSizeInBits() >= 8 &&
1373 isPowerOf2_64(Value: Query.Types[0].getScalarSizeInBits()) &&
1374 Query.Types[0].getSizeInBits() <= 128 &&
1375 Query.Types[1].getSizeInBits() <= 64;
1376 },
1377 Mutation: [=](const LegalityQuery &Query) {
1378 const LLT DstTy = Query.Types[0];
1379 const LLT SrcTy = Query.Types[1];
1380 return std::pair(
1381 0, DstTy.changeElementSize(NewEltSize: SrcTy.getSizeInBits())
1382 .changeElementCount(
1383 EC: DstTy.getElementCount().divideCoefficientBy(
1384 RHS: SrcTy.getNumElements())));
1385 });
1386
1387 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
1388 .legalFor(Types: {{v8s8, v16s8}, {v4s16, v8s16}, {v2s32, v4s32}})
1389 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1390 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
1391 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
1392 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
1393 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
1394 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
1395 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
1396 .lower()
1397 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1398
1399 // TODO: {nxv16s8, s8}, {nxv8s16, s16}
1400 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
1401 .legalFor(Pred: HasSVE, Types: {{nxv4s32, s32}, {nxv2s64, s64}});
1402
1403 getActionDefinitionsBuilder(Opcode: G_JUMP_TABLE).legalFor(Types: {p0});
1404
1405 getActionDefinitionsBuilder(Opcode: G_BRJT).legalFor(Types: {{p0, s64}});
1406
1407 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
1408
1409 getActionDefinitionsBuilder(Opcode: G_DYN_STACKALLOC).custom();
1410
1411 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).lower();
1412
1413 if (ST.hasMOPS()) {
1414 // G_BZERO is not supported. Currently it is only emitted by
1415 // PreLegalizerCombiner for G_MEMSET with zero constant.
1416 getActionDefinitionsBuilder(Opcode: G_BZERO).unsupported();
1417
1418 getActionDefinitionsBuilder(Opcode: G_MEMSET)
1419 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1420 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64})
1421 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1422
1423 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE})
1424 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64})
1425 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1426
1427 // G_MEMCPY_INLINE does not have a tailcall immediate
1428 getActionDefinitionsBuilder(Opcode: G_MEMCPY_INLINE)
1429 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64});
1430
1431 getActionDefinitionsBuilder(Opcode: G_MEMSET_INLINE)
1432 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1433 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64});
1434 } else {
1435 getActionDefinitionsBuilder(Opcodes: {G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
1436 .libcall();
1437 }
1438
1439 // For fadd reductions we have pairwise operations available. We treat the
1440 // usual legal types as legal and handle the lowering to pairwise instructions
1441 // later.
1442 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FADD)
1443 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1444 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1445 .widenScalarIf(
1446 Predicate: [HasFP16](const LegalityQuery &Query) {
1447 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1448 Query.Types[0].getScalarType().isBFloat16();
1449 },
1450 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1451 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1452 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1453 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1454 .moreElementsToNextPow2(TypeIdx: 1)
1455 .scalarize(TypeIdx: 1)
1456 .lower();
1457
1458 // For fmul reductions we need to split up into individual operations. We
1459 // clamp to 128 bit vectors then to 64bit vectors to produce a cascade of
1460 // smaller types, followed by scalarizing what remains.
1461 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FMUL)
1462 .widenScalarIf(
1463 Predicate: [HasFP16](const LegalityQuery &Query) {
1464 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1465 Query.Types[0].getScalarType().isBFloat16();
1466 },
1467 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1468 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1469 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1470 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1471 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1472 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1473 .scalarize(TypeIdx: 1)
1474 .lower();
1475
1476 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
1477 .scalarize(TypeIdx: 2)
1478 .lower();
1479
1480 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_ADD)
1481 .legalFor(Types: {{i8, v8i8},
1482 {i8, v16i8},
1483 {i16, v4i16},
1484 {i16, v8i16},
1485 {i32, v2i32},
1486 {i32, v4i32},
1487 {i64, v2i64}})
1488 .moreElementsToNextPow2(TypeIdx: 1)
1489 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1490 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1491 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1492 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1493 .widenVectorEltsToVectorMinSize(TypeIdx: 1, VectorSize: 64)
1494 .scalarize(TypeIdx: 1);
1495
1496 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
1497 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1498 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1499 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1500 .widenScalarIf(
1501 Predicate: [HasFP16](const LegalityQuery &Query) {
1502 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1503 Query.Types[0].getScalarType().isBFloat16();
1504 },
1505 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1506 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1507 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1508 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1509 .scalarize(TypeIdx: 1)
1510 .lower();
1511
1512 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_MUL)
1513 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1514 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1515 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
1516 .scalarize(TypeIdx: 1)
1517 .lower();
1518
1519 getActionDefinitionsBuilder(
1520 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1521 .legalFor(Types: {{i8, v8i8},
1522 {i8, v16i8},
1523 {i16, v4i16},
1524 {i16, v8i16},
1525 {i32, v2i32},
1526 {i32, v4i32}})
1527 .moreElementsIf(
1528 Predicate: [=](const LegalityQuery &Query) {
1529 return Query.Types[1].isVector() &&
1530 Query.Types[1].getElementType() != s8 &&
1531 Query.Types[1].getNumElements() & 1;
1532 },
1533 Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx: 1))
1534 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1535 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1536 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1537 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1538 .scalarize(TypeIdx: 1)
1539 .lower();
1540
1541 getActionDefinitionsBuilder(
1542 Opcodes: {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1543 // Try to break down into smaller vectors as long as they're at least 64
1544 // bits. This lets us use vector operations for some parts of the
1545 // reduction.
1546 .fewerElementsIf(
1547 Predicate: [=](const LegalityQuery &Q) {
1548 LLT SrcTy = Q.Types[1];
1549 if (SrcTy.isScalar())
1550 return false;
1551 if (!isPowerOf2_32(Value: SrcTy.getNumElements()))
1552 return false;
1553 // We can usually perform 64b vector operations.
1554 return SrcTy.getSizeInBits() > 64;
1555 },
1556 Mutation: [=](const LegalityQuery &Q) {
1557 LLT SrcTy = Q.Types[1];
1558 return std::make_pair(x: 1, y: SrcTy.divide(Factor: 2));
1559 })
1560 .scalarize(TypeIdx: 1)
1561 .lower();
1562
1563 // TODO: Update this to correct handling when adding AArch64/SVE support.
1564 getActionDefinitionsBuilder(Opcode: G_VECTOR_COMPRESS).lower();
1565
1566 // Access to floating-point environment.
1567 getActionDefinitionsBuilder(Opcodes: {G_GET_FPENV, G_SET_FPENV, G_RESET_FPENV,
1568 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1569 .libcall();
1570
1571 getActionDefinitionsBuilder(Opcodes: {G_GET_ROUNDING, G_SET_ROUNDING})
1572 .customFor(Types: {s32});
1573
1574 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).lower();
1575
1576 getActionDefinitionsBuilder(Opcode: G_PREFETCH).custom();
1577
1578 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
1579
1580 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
1581 .alwaysLegal();
1582 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
1583 getActionDefinitionsBuilder(Opcode: G_INVOKE_REGION_START).alwaysLegal();
1584
1585 verify(MII: *ST.getInstrInfo());
1586}
1587
1588bool AArch64LegalizerInfo::legalizeCustom(
1589 LegalizerHelper &Helper, MachineInstr &MI,
1590 LostDebugLocObserver &LocObserver) const {
1591 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1592 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1593 GISelChangeObserver &Observer = Helper.Observer;
1594 switch (MI.getOpcode()) {
1595 default:
1596 // No idea what to do.
1597 return false;
1598 case TargetOpcode::G_VAARG:
1599 return legalizeVaArg(MI, MRI, MIRBuilder);
1600 case TargetOpcode::G_LOAD:
1601 case TargetOpcode::G_STORE:
1602 return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
1603 case TargetOpcode::G_SHL:
1604 case TargetOpcode::G_ASHR:
1605 case TargetOpcode::G_LSHR:
1606 return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
1607 case TargetOpcode::G_GLOBAL_VALUE:
1608 return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
1609 case TargetOpcode::G_SBFX:
1610 case TargetOpcode::G_UBFX:
1611 return legalizeBitfieldExtract(MI, MRI, Helper);
1612 case TargetOpcode::G_FSHL:
1613 case TargetOpcode::G_FSHR:
1614 return legalizeFunnelShift(MI, MRI, MIRBuilder, Observer, Helper);
1615 case TargetOpcode::G_ROTR:
1616 return legalizeRotate(MI, MRI, Helper);
1617 case TargetOpcode::G_CTPOP:
1618 return legalizeCTPOP(MI, MRI, Helper);
1619 case TargetOpcode::G_ATOMIC_CMPXCHG:
1620 return legalizeAtomicCmpxchg128(MI, MRI, Helper);
1621 case TargetOpcode::G_CTTZ:
1622 return legalizeCTTZ(MI, Helper);
1623 case TargetOpcode::G_BZERO:
1624 case TargetOpcode::G_MEMCPY:
1625 case TargetOpcode::G_MEMMOVE:
1626 case TargetOpcode::G_MEMSET:
1627 case TargetOpcode::G_MEMSET_INLINE:
1628 return legalizeMemOps(MI, Helper);
1629 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1630 return legalizeExtractVectorElt(MI, MRI, Helper);
1631 case TargetOpcode::G_DYN_STACKALLOC:
1632 return legalizeDynStackAlloc(MI, Helper);
1633 case TargetOpcode::G_PREFETCH:
1634 return legalizePrefetch(MI, Helper);
1635 case TargetOpcode::G_ABS:
1636 return Helper.lowerAbsToCNeg(MI);
1637 case TargetOpcode::G_ICMP:
1638 return legalizeICMP(MI, MRI, MIRBuilder);
1639 case TargetOpcode::G_BITCAST:
1640 return legalizeBitcast(MI, Helper);
1641 case TargetOpcode::G_CONCAT_VECTORS:
1642 return legalizeConcatVectors(MI, MRI, MIRBuilder);
1643 case TargetOpcode::G_FPTRUNC:
1644 // In order to lower f16 to f64 properly, we need to use f32 as an
1645 // intermediary
1646 return legalizeFptrunc(MI, MIRBuilder, MRI);
1647 case TargetOpcode::G_GET_ROUNDING:
1648 return legalizeGetRounding(MI, MIRBuilder, MRI, Helper);
1649 case TargetOpcode::G_SET_ROUNDING:
1650 return legalizeSetRounding(MI, MIRBuilder, MRI, Helper);
1651 }
1652
1653 llvm_unreachable("expected switch to return");
1654}
1655
1656bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
1657 LegalizerHelper &Helper) const {
1658 assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
1659 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1660 // We're trying to handle casts from i1 vectors to scalars but reloading from
1661 // stack.
1662 if (!DstTy.isScalar() || !SrcTy.isVector() ||
1663 SrcTy.getElementType() != LLT::scalar(SizeInBits: 1))
1664 return false;
1665
1666 Helper.createStackStoreLoad(Res: DstReg, Val: SrcReg);
1667 MI.eraseFromParent();
1668 return true;
1669}
1670
1671bool AArch64LegalizerInfo::legalizeFunnelShift(MachineInstr &MI,
1672 MachineRegisterInfo &MRI,
1673 MachineIRBuilder &MIRBuilder,
1674 GISelChangeObserver &Observer,
1675 LegalizerHelper &Helper) const {
1676 assert(MI.getOpcode() == TargetOpcode::G_FSHL ||
1677 MI.getOpcode() == TargetOpcode::G_FSHR);
1678
1679 // Keep as G_FSHR if shift amount is a G_CONSTANT, else use generic
1680 // lowering
1681 Register ShiftNo = MI.getOperand(i: 3).getReg();
1682 LLT ShiftTy = MRI.getType(Reg: ShiftNo);
1683 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ShiftNo, MRI);
1684
1685 // Adjust shift amount according to Opcode (FSHL/FSHR)
1686 // Convert FSHL to FSHR
1687 LLT OperationTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1688 APInt BitWidth(ShiftTy.getSizeInBits(), OperationTy.getSizeInBits(), false);
1689
1690 // Lower non-constant shifts and leave zero shifts to the optimizer.
1691 if (!VRegAndVal || VRegAndVal->Value.urem(RHS: BitWidth) == 0)
1692 return (Helper.lowerFunnelShiftAsShifts(MI) ==
1693 LegalizerHelper::LegalizeResult::Legalized);
1694
1695 APInt Amount = VRegAndVal->Value.urem(RHS: BitWidth);
1696
1697 Amount = MI.getOpcode() == TargetOpcode::G_FSHL ? BitWidth - Amount : Amount;
1698
1699 // If the instruction is G_FSHR, has a 64-bit G_CONSTANT for shift amount
1700 // in the range of 0 <-> BitWidth, it is legal
1701 if (ShiftTy.getSizeInBits() == 64 && MI.getOpcode() == TargetOpcode::G_FSHR &&
1702 VRegAndVal->Value.ult(RHS: BitWidth))
1703 return true;
1704
1705 // Cast the ShiftNumber to a 64-bit type
1706 auto Cast64 = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Amount.zext(width: 64));
1707
1708 if (MI.getOpcode() == TargetOpcode::G_FSHR) {
1709 Observer.changingInstr(MI);
1710 MI.getOperand(i: 3).setReg(Cast64.getReg(Idx: 0));
1711 Observer.changedInstr(MI);
1712 }
1713 // If Opcode is FSHL, remove the FSHL instruction and create a FSHR
1714 // instruction
1715 else if (MI.getOpcode() == TargetOpcode::G_FSHL) {
1716 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FSHR, DstOps: {MI.getOperand(i: 0).getReg()},
1717 SrcOps: {MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getReg(),
1718 Cast64.getReg(Idx: 0)});
1719 MI.eraseFromParent();
1720 }
1721 return true;
1722}
1723
1724bool AArch64LegalizerInfo::legalizeICMP(MachineInstr &MI,
1725 MachineRegisterInfo &MRI,
1726 MachineIRBuilder &MIRBuilder) const {
1727 Register DstReg = MI.getOperand(i: 0).getReg();
1728 Register SrcReg1 = MI.getOperand(i: 2).getReg();
1729 Register SrcReg2 = MI.getOperand(i: 3).getReg();
1730 LLT DstTy = MRI.getType(Reg: DstReg);
1731 LLT SrcTy = MRI.getType(Reg: SrcReg1);
1732
1733 // Check the vector types are legal
1734 if (DstTy.getScalarSizeInBits() != SrcTy.getScalarSizeInBits() ||
1735 DstTy.getNumElements() != SrcTy.getNumElements() ||
1736 (DstTy.getSizeInBits() != 64 && DstTy.getSizeInBits() != 128))
1737 return false;
1738
1739 // Lowers G_ICMP NE => G_ICMP EQ to allow better pattern matching for
1740 // following passes
1741 CmpInst::Predicate Pred = (CmpInst::Predicate)MI.getOperand(i: 1).getPredicate();
1742 if (Pred != CmpInst::ICMP_NE)
1743 return true;
1744 Register CmpReg =
1745 MIRBuilder
1746 .buildICmp(Pred: CmpInst::ICMP_EQ, Res: MRI.getType(Reg: DstReg), Op0: SrcReg1, Op1: SrcReg2)
1747 .getReg(Idx: 0);
1748 MIRBuilder.buildNot(Dst: DstReg, Src0: CmpReg);
1749
1750 MI.eraseFromParent();
1751 return true;
1752}
1753
1754bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
1755 MachineRegisterInfo &MRI,
1756 LegalizerHelper &Helper) const {
1757 // To allow for imported patterns to match, we ensure that the rotate amount
1758 // is 64b with an extension.
1759 Register AmtReg = MI.getOperand(i: 2).getReg();
1760 LLT AmtTy = MRI.getType(Reg: AmtReg);
1761 (void)AmtTy;
1762 assert(AmtTy.isScalar() && "Expected a scalar rotate");
1763 assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
1764 auto NewAmt = Helper.MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 64), Op: AmtReg);
1765 Helper.Observer.changingInstr(MI);
1766 MI.getOperand(i: 2).setReg(NewAmt.getReg(Idx: 0));
1767 Helper.Observer.changedInstr(MI);
1768 return true;
1769}
1770
1771bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1772 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
1773 GISelChangeObserver &Observer) const {
1774 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1775 // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
1776 // G_ADD_LOW instructions.
1777 // By splitting this here, we can optimize accesses in the small code model by
1778 // folding in the G_ADD_LOW into the load/store offset.
1779 auto &GlobalOp = MI.getOperand(i: 1);
1780 // Don't modify an intrinsic call.
1781 if (GlobalOp.isSymbol())
1782 return true;
1783 const auto* GV = GlobalOp.getGlobal();
1784 if (GV->isThreadLocal())
1785 return true; // Don't want to modify TLS vars.
1786
1787 auto &TM = ST->getTargetLowering()->getTargetMachine();
1788 unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
1789
1790 if (OpFlags & AArch64II::MO_GOT)
1791 return true;
1792
1793 auto Offset = GlobalOp.getOffset();
1794 Register DstReg = MI.getOperand(i: 0).getReg();
1795 auto ADRP = MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {})
1796 .addGlobalAddress(GV, Offset, TargetFlags: OpFlags | AArch64II::MO_PAGE);
1797 // Set the regclass on the dest reg too.
1798 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1799
1800 // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
1801 // by creating a MOVK that sets bits 48-63 of the register to (global address
1802 // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
1803 // prevent an incorrect tag being generated during relocation when the
1804 // global appears before the code section. Without the offset, a global at
1805 // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
1806 // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
1807 // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
1808 // instead of `0xf`.
1809 // This assumes that we're in the small code model so we can assume a binary
1810 // size of <= 4GB, which makes the untagged PC relative offset positive. The
1811 // binary must also be loaded into address range [0, 2^48). Both of these
1812 // properties need to be ensured at runtime when using tagged addresses.
1813 if (OpFlags & AArch64II::MO_TAGGED) {
1814 assert(!Offset &&
1815 "Should not have folded in an offset for a tagged global!");
1816 ADRP = MIRBuilder.buildInstr(Opc: AArch64::MOVKXi, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {ADRP})
1817 .addGlobalAddress(GV, Offset: 0x100000000,
1818 TargetFlags: AArch64II::MO_PREL | AArch64II::MO_G3)
1819 .addImm(Val: 48);
1820 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1821 }
1822
1823 MIRBuilder.buildInstr(Opc: AArch64::G_ADD_LOW, DstOps: {DstReg}, SrcOps: {ADRP})
1824 .addGlobalAddress(GV, Offset,
1825 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1826 MI.eraseFromParent();
1827 return true;
1828}
1829
1830bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1831 MachineInstr &MI) const {
1832 MachineIRBuilder &MIB = Helper.MIRBuilder;
1833 MachineRegisterInfo &MRI = *MIB.getMRI();
1834
1835 auto LowerUnaryOp = [&MI, &MIB](unsigned Opcode) {
1836 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1837 MI.eraseFromParent();
1838 return true;
1839 };
1840 auto LowerBinOp = [&MI, &MIB](unsigned Opcode) {
1841 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1842 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
1843 MI.eraseFromParent();
1844 return true;
1845 };
1846 auto LowerTriOp = [&MI, &MIB](unsigned Opcode) {
1847 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1848 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4)});
1849 MI.eraseFromParent();
1850 return true;
1851 };
1852
1853 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1854 switch (IntrinsicID) {
1855 case Intrinsic::vacopy: {
1856 unsigned PtrSize = ST->isTargetILP32() ? 4 : 8;
1857 unsigned VaListSize =
1858 (ST->isTargetDarwin() || ST->isTargetWindows())
1859 ? PtrSize
1860 : ST->isTargetILP32() ? 20 : 32;
1861
1862 MachineFunction &MF = *MI.getMF();
1863 auto Val = MF.getRegInfo().createGenericVirtualRegister(
1864 Ty: LLT::integer(SizeInBits: VaListSize * 8));
1865 MIB.buildLoad(Res: Val, Addr: MI.getOperand(i: 2),
1866 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1867 F: MachineMemOperand::MOLoad,
1868 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1869 MIB.buildStore(Val, Addr: MI.getOperand(i: 1),
1870 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1871 F: MachineMemOperand::MOStore,
1872 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1873 MI.eraseFromParent();
1874 return true;
1875 }
1876 case Intrinsic::get_dynamic_area_offset: {
1877 MIB.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: 0);
1878 MI.eraseFromParent();
1879 return true;
1880 }
1881 case Intrinsic::aarch64_mops_memset_tag: {
1882 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1883 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
1884 // the instruction).
1885 auto &Value = MI.getOperand(i: 3);
1886 Register ExtValueReg = MIB.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
1887 Value.setReg(ExtValueReg);
1888 return true;
1889 }
1890 case Intrinsic::aarch64_prefetch: {
1891 auto &AddrVal = MI.getOperand(i: 1);
1892
1893 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1894 int64_t Target = MI.getOperand(i: 3).getImm();
1895 int64_t IsStream = MI.getOperand(i: 4).getImm();
1896 int64_t IsData = MI.getOperand(i: 5).getImm();
1897
1898 unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
1899 (!IsData << 3) | // IsDataCache bit
1900 (Target << 1) | // Cache level bits
1901 (unsigned)IsStream; // Stream bit
1902
1903 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
1904 MI.eraseFromParent();
1905 return true;
1906 }
1907 case Intrinsic::aarch64_range_prefetch: {
1908 auto &AddrVal = MI.getOperand(i: 1);
1909
1910 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1911 int64_t IsStream = MI.getOperand(i: 3).getImm();
1912 unsigned PrfOp = (IsStream << 2) | IsWrite;
1913
1914 MIB.buildInstr(Opcode: AArch64::G_AARCH64_RANGE_PREFETCH)
1915 .addImm(Val: PrfOp)
1916 .add(MO: AddrVal)
1917 .addUse(RegNo: MI.getOperand(i: 4).getReg()); // Metadata
1918 MI.eraseFromParent();
1919 return true;
1920 }
1921 case Intrinsic::aarch64_prefetch_ir: {
1922 auto &AddrVal = MI.getOperand(i: 1);
1923 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: 24).add(MO: AddrVal);
1924 MI.eraseFromParent();
1925 return true;
1926 }
1927 case Intrinsic::aarch64_neon_uaddv:
1928 case Intrinsic::aarch64_neon_saddv:
1929 case Intrinsic::aarch64_neon_umaxv:
1930 case Intrinsic::aarch64_neon_smaxv:
1931 case Intrinsic::aarch64_neon_uminv:
1932 case Intrinsic::aarch64_neon_sminv: {
1933 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1934 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1935 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1936
1937 auto OldDst = MI.getOperand(i: 0).getReg();
1938 auto OldDstTy = MRI.getType(Reg: OldDst);
1939 LLT NewDstTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType();
1940 if (OldDstTy == NewDstTy)
1941 return true;
1942
1943 auto NewDst = MRI.createGenericVirtualRegister(Ty: NewDstTy);
1944
1945 Helper.Observer.changingInstr(MI);
1946 MI.getOperand(i: 0).setReg(NewDst);
1947 Helper.Observer.changedInstr(MI);
1948
1949 MIB.setInsertPt(MBB&: MIB.getMBB(), II: ++MIB.getInsertPt());
1950 MIB.buildExtOrTrunc(ExtOpc: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1951 Res: OldDst, Op: NewDst);
1952
1953 return true;
1954 }
1955 case Intrinsic::aarch64_neon_uaddlp:
1956 case Intrinsic::aarch64_neon_saddlp: {
1957 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1958 ? AArch64::G_UADDLP
1959 : AArch64::G_SADDLP;
1960 MIB.buildInstr(Opc, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1961 MI.eraseFromParent();
1962
1963 return true;
1964 }
1965 case Intrinsic::aarch64_neon_uaddlv:
1966 case Intrinsic::aarch64_neon_saddlv: {
1967 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1968 ? AArch64::G_UADDLV
1969 : AArch64::G_SADDLV;
1970 Register DstReg = MI.getOperand(i: 0).getReg();
1971 Register SrcReg = MI.getOperand(i: 2).getReg();
1972 LLT DstTy = MRI.getType(Reg: DstReg);
1973
1974 LLT MidTy, ExtTy;
1975 if (DstTy.isScalar() && DstTy.getScalarSizeInBits() <= 32) {
1976 ExtTy = LLT::integer(SizeInBits: 32);
1977 MidTy = LLT::fixed_vector(NumElements: 4, ScalarTy: ExtTy);
1978 } else {
1979 ExtTy = LLT::integer(SizeInBits: 64);
1980 MidTy = LLT::fixed_vector(NumElements: 2, ScalarTy: ExtTy);
1981 }
1982
1983 Register MidReg =
1984 MIB.buildInstr(Opc, DstOps: {MidTy}, SrcOps: {SrcReg})->getOperand(i: 0).getReg();
1985 Register ZeroReg =
1986 MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0)->getOperand(i: 0).getReg();
1987 Register ExtReg = MIB.buildInstr(Opc: AArch64::G_EXTRACT_VECTOR_ELT, DstOps: {ExtTy},
1988 SrcOps: {MidReg, ZeroReg})
1989 .getReg(Idx: 0);
1990
1991 if (DstTy.getScalarSizeInBits() < 32)
1992 MIB.buildTrunc(Res: DstReg, Op: ExtReg);
1993 else
1994 MIB.buildCopy(Res: DstReg, Op: ExtReg);
1995
1996 MI.eraseFromParent();
1997
1998 return true;
1999 }
2000 case Intrinsic::aarch64_neon_fmax:
2001 return LowerBinOp(TargetOpcode::G_FMAXIMUM);
2002 case Intrinsic::aarch64_neon_fmin:
2003 return LowerBinOp(TargetOpcode::G_FMINIMUM);
2004 case Intrinsic::aarch64_neon_fmaxnm:
2005 return LowerBinOp(TargetOpcode::G_FMAXNUM);
2006 case Intrinsic::aarch64_neon_fminnm:
2007 return LowerBinOp(TargetOpcode::G_FMINNUM);
2008 case Intrinsic::aarch64_neon_pmul:
2009 return LowerBinOp(TargetOpcode::G_CLMUL);
2010 case Intrinsic::aarch64_neon_pmull:
2011 case Intrinsic::aarch64_neon_pmull64:
2012 return LowerBinOp(AArch64::G_PMULL);
2013 case Intrinsic::aarch64_neon_smull:
2014 return LowerBinOp(AArch64::G_SMULL);
2015 case Intrinsic::aarch64_neon_umull:
2016 return LowerBinOp(AArch64::G_UMULL);
2017 case Intrinsic::aarch64_neon_sabd:
2018 return LowerBinOp(TargetOpcode::G_ABDS);
2019 case Intrinsic::aarch64_neon_uabd:
2020 return LowerBinOp(TargetOpcode::G_ABDU);
2021 case Intrinsic::aarch64_neon_uhadd:
2022 return LowerBinOp(TargetOpcode::G_UAVGFLOOR);
2023 case Intrinsic::aarch64_neon_urhadd:
2024 return LowerBinOp(TargetOpcode::G_UAVGCEIL);
2025 case Intrinsic::aarch64_neon_shadd:
2026 return LowerBinOp(TargetOpcode::G_SAVGFLOOR);
2027 case Intrinsic::aarch64_neon_srhadd:
2028 return LowerBinOp(TargetOpcode::G_SAVGCEIL);
2029 case Intrinsic::aarch64_neon_sqshrn: {
2030 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2031 return true;
2032 // Create right shift instruction. Store the output register in Shr.
2033 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2034 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2035 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2036 // Build the narrow intrinsic, taking in Shr.
2037 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2038 MI.eraseFromParent();
2039 return true;
2040 }
2041 case Intrinsic::aarch64_neon_sqshrun: {
2042 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2043 return true;
2044 // Create right shift instruction. Store the output register in Shr.
2045 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2046 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2047 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2048 // Build the narrow intrinsic, taking in Shr.
2049 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2050 MI.eraseFromParent();
2051 return true;
2052 }
2053 case Intrinsic::aarch64_neon_sqrshrn: {
2054 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2055 return true;
2056 // Create right shift instruction. Store the output register in Shr.
2057 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2058 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2059 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2060 // Build the narrow intrinsic, taking in Shr.
2061 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2062 MI.eraseFromParent();
2063 return true;
2064 }
2065 case Intrinsic::aarch64_neon_sqrshrun: {
2066 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2067 return true;
2068 // Create right shift instruction. Store the output register in Shr.
2069 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2070 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2071 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2072 // Build the narrow intrinsic, taking in Shr.
2073 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2074 MI.eraseFromParent();
2075 return true;
2076 }
2077 case Intrinsic::aarch64_neon_uqrshrn: {
2078 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2079 return true;
2080 // Create right shift instruction. Store the output register in Shr.
2081 auto Shr = MIB.buildInstr(Opc: AArch64::G_URSHR_I,
2082 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2083 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2084 // Build the narrow intrinsic, taking in Shr.
2085 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2086 MI.eraseFromParent();
2087 return true;
2088 }
2089 case Intrinsic::aarch64_neon_uqshrn: {
2090 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2091 return true;
2092 // Create right shift instruction. Store the output register in Shr.
2093 auto Shr = MIB.buildInstr(Opc: AArch64::G_VLSHR,
2094 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2095 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2096 // Build the narrow intrinsic, taking in Shr.
2097 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2098 MI.eraseFromParent();
2099 return true;
2100 }
2101 case Intrinsic::aarch64_neon_sqshlu: {
2102 // Check if last operand is constant vector dup
2103 auto ShiftAmount =
2104 isConstantOrConstantSplatVector(Def: MI.getOperand(i: 3).getReg(), MRI);
2105 if (ShiftAmount) {
2106 // If so, create a new intrinsic with the correct shift amount
2107 MIB.buildInstr(Opc: AArch64::G_SQSHLU_I, DstOps: {MI.getOperand(i: 0)},
2108 SrcOps: {MI.getOperand(i: 2)})
2109 .addImm(Val: ShiftAmount->getSExtValue());
2110 MI.eraseFromParent();
2111 return true;
2112 }
2113 return false;
2114 }
2115 case Intrinsic::aarch64_neon_vsli: {
2116 MIB.buildInstr(
2117 Opc: AArch64::G_SLI, DstOps: {MI.getOperand(i: 0)},
2118 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2119 MI.eraseFromParent();
2120 break;
2121 }
2122 case Intrinsic::aarch64_neon_vsri: {
2123 MIB.buildInstr(
2124 Opc: AArch64::G_SRI, DstOps: {MI.getOperand(i: 0)},
2125 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2126 MI.eraseFromParent();
2127 break;
2128 }
2129 case Intrinsic::aarch64_neon_abs: {
2130 // Lower the intrinsic to G_ABS.
2131 MIB.buildInstr(Opc: TargetOpcode::G_ABS, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
2132 MI.eraseFromParent();
2133 return true;
2134 }
2135 case Intrinsic::aarch64_neon_addhn:
2136 return LowerBinOp(AArch64::G_ADDHN);
2137 case Intrinsic::aarch64_neon_sqadd: {
2138 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2139 return LowerBinOp(TargetOpcode::G_SADDSAT);
2140 break;
2141 }
2142 case Intrinsic::aarch64_neon_sqsub: {
2143 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2144 return LowerBinOp(TargetOpcode::G_SSUBSAT);
2145 break;
2146 }
2147 case Intrinsic::aarch64_neon_uqadd: {
2148 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2149 return LowerBinOp(TargetOpcode::G_UADDSAT);
2150 break;
2151 }
2152 case Intrinsic::aarch64_neon_uqsub: {
2153 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2154 return LowerBinOp(TargetOpcode::G_USUBSAT);
2155 break;
2156 }
2157 case Intrinsic::aarch64_neon_udot:
2158 return LowerTriOp(AArch64::G_UDOT);
2159 case Intrinsic::aarch64_neon_sdot:
2160 return LowerTriOp(AArch64::G_SDOT);
2161 case Intrinsic::aarch64_neon_usdot:
2162 return LowerTriOp(AArch64::G_USDOT);
2163 case Intrinsic::aarch64_neon_sqxtn:
2164 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_S);
2165 case Intrinsic::aarch64_neon_sqxtun:
2166 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_U);
2167 case Intrinsic::aarch64_neon_uqxtn:
2168 return LowerUnaryOp(TargetOpcode::G_TRUNC_USAT_U);
2169 case Intrinsic::aarch64_neon_fcvtzu:
2170 return LowerUnaryOp(TargetOpcode::G_FPTOUI_SAT);
2171 case Intrinsic::aarch64_neon_fcvtzs:
2172 return LowerUnaryOp(TargetOpcode::G_FPTOSI_SAT);
2173 case Intrinsic::aarch64_neon_cls:
2174 return LowerUnaryOp(TargetOpcode::G_CTLS);
2175
2176 case Intrinsic::vector_reverse:
2177 // TODO: Add support for vector_reverse
2178 return false;
2179 }
2180
2181 return true;
2182}
2183
2184bool AArch64LegalizerInfo::legalizeShlAshrLshr(
2185 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2186 GISelChangeObserver &Observer) const {
2187 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
2188 MI.getOpcode() == TargetOpcode::G_LSHR ||
2189 MI.getOpcode() == TargetOpcode::G_SHL);
2190 // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
2191 // imported patterns can select it later. Either way, it will be legal.
2192 Register AmtReg = MI.getOperand(i: 2).getReg();
2193 LLT AmtRegEltTy = MRI.getType(Reg: AmtReg).getScalarType();
2194 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI);
2195 if (!VRegAndVal)
2196 return true;
2197 // Check the shift amount is in range for an immediate form.
2198 int64_t Amount = VRegAndVal->Value.getSExtValue();
2199 if (Amount > 31)
2200 return true; // This will have to remain a register variant.
2201 auto ExtCst =
2202 MIRBuilder.buildConstant(Res: AmtRegEltTy.changeElementSize(NewEltSize: 64), Val: Amount);
2203 Observer.changingInstr(MI);
2204 MI.getOperand(i: 2).setReg(ExtCst.getReg(Idx: 0));
2205 Observer.changedInstr(MI);
2206 return true;
2207}
2208
2209static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset,
2210 MachineRegisterInfo &MRI) {
2211 Base = Root;
2212 Offset = 0;
2213
2214 Register NewBase;
2215 int64_t NewOffset;
2216 if (mi_match(R: Root, MRI, P: m_GPtrAdd(L: m_Reg(R&: NewBase), R: m_ICst(Cst&: NewOffset))) &&
2217 isShiftedInt<7, 3>(x: NewOffset)) {
2218 Base = NewBase;
2219 Offset = NewOffset;
2220 }
2221}
2222
2223// FIXME: This should be removed and replaced with the generic bitcast legalize
2224// action.
2225bool AArch64LegalizerInfo::legalizeLoadStore(
2226 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2227 GISelChangeObserver &Observer) const {
2228 assert(MI.getOpcode() == TargetOpcode::G_STORE ||
2229 MI.getOpcode() == TargetOpcode::G_LOAD);
2230 // Here we just try to handle vector loads/stores where our value type might
2231 // have pointer elements, which the SelectionDAG importer can't handle. To
2232 // allow the existing patterns for s64 to fire for p0, we just try to bitcast
2233 // the value to use s64 types.
2234
2235 // Custom legalization requires the instruction, if not deleted, must be fully
2236 // legalized. In order to allow further legalization of the inst, we create
2237 // a new instruction and erase the existing one.
2238
2239 Register ValReg = MI.getOperand(i: 0).getReg();
2240 const LLT ValTy = MRI.getType(Reg: ValReg);
2241
2242 if (ValTy == LLT::scalar(SizeInBits: 128)) {
2243
2244 AtomicOrdering Ordering = (*MI.memoperands_begin())->getSuccessOrdering();
2245 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2246 bool IsLoadAcquire = IsLoad && Ordering == AtomicOrdering::Acquire;
2247 bool IsStoreRelease = !IsLoad && Ordering == AtomicOrdering::Release;
2248 bool IsRcpC3 =
2249 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
2250
2251 LLT s64 = LLT::integer(SizeInBits: 64);
2252
2253 unsigned Opcode;
2254 if (IsRcpC3) {
2255 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
2256 } else {
2257 // For LSE2, loads/stores should have been converted to monotonic and had
2258 // a fence inserted after them.
2259 assert(Ordering == AtomicOrdering::Monotonic ||
2260 Ordering == AtomicOrdering::Unordered);
2261 assert(ST->hasLSE2() && "ldp/stp not single copy atomic without +lse2");
2262
2263 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
2264 }
2265
2266 MachineInstrBuilder NewI;
2267 if (IsLoad) {
2268 NewI = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {s64, s64}, SrcOps: {});
2269 MIRBuilder.buildMergeLikeInstr(
2270 Res: ValReg, Ops: {NewI->getOperand(i: 0), NewI->getOperand(i: 1)});
2271 } else {
2272 auto Split = MIRBuilder.buildUnmerge(Res: s64, Op: MI.getOperand(i: 0));
2273 NewI = MIRBuilder.buildInstr(
2274 Opc: Opcode, DstOps: {}, SrcOps: {Split->getOperand(i: 0), Split->getOperand(i: 1)});
2275 }
2276
2277 if (IsRcpC3) {
2278 NewI.addUse(RegNo: MI.getOperand(i: 1).getReg());
2279 } else {
2280 Register Base;
2281 int Offset;
2282 matchLDPSTPAddrMode(Root: MI.getOperand(i: 1).getReg(), Base, Offset, MRI);
2283 NewI.addUse(RegNo: Base);
2284 NewI.addImm(Val: Offset / 8);
2285 }
2286
2287 NewI.cloneMemRefs(OtherMI: MI);
2288 constrainSelectedInstRegOperands(I&: *NewI, TII: *ST->getInstrInfo(),
2289 TRI: *MRI.getTargetRegisterInfo(),
2290 RBI: *ST->getRegBankInfo());
2291 MI.eraseFromParent();
2292 return true;
2293 }
2294
2295 if (!ValTy.isPointerVector() ||
2296 ValTy.getElementType().getAddressSpace() != 0) {
2297 LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
2298 return false;
2299 }
2300
2301 unsigned PtrSize = ValTy.getElementType().getSizeInBits();
2302 const LLT NewTy = LLT::vector(EC: ValTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: PtrSize));
2303 auto &MMO = **MI.memoperands_begin();
2304 MMO.setType(NewTy);
2305
2306 if (MI.getOpcode() == TargetOpcode::G_STORE) {
2307 auto Bitcast = MIRBuilder.buildBitcast(Dst: NewTy, Src: ValReg);
2308 MIRBuilder.buildStore(Val: Bitcast.getReg(Idx: 0), Addr: MI.getOperand(i: 1), MMO);
2309 } else {
2310 auto NewLoad = MIRBuilder.buildLoad(Res: NewTy, Addr: MI.getOperand(i: 1), MMO);
2311 MIRBuilder.buildBitcast(Dst: ValReg, Src: NewLoad);
2312 }
2313 MI.eraseFromParent();
2314 return true;
2315}
2316
2317bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
2318 MachineRegisterInfo &MRI,
2319 MachineIRBuilder &MIRBuilder) const {
2320 MachineFunction &MF = MIRBuilder.getMF();
2321 Align Alignment(MI.getOperand(i: 2).getImm());
2322 Register Dst = MI.getOperand(i: 0).getReg();
2323 Register ListPtr = MI.getOperand(i: 1).getReg();
2324
2325 LLT PtrTy = MRI.getType(Reg: ListPtr);
2326 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
2327
2328 const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
2329 const Align PtrAlign = Align(PtrSize);
2330 auto List = MIRBuilder.buildLoad(
2331 Res: PtrTy, Addr: ListPtr,
2332 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2333 MemTy: PtrTy, BaseAlignment: PtrAlign));
2334
2335 MachineInstrBuilder DstPtr;
2336 if (Alignment > PtrAlign) {
2337 // Realign the list to the actual required alignment.
2338 auto AlignMinus1 =
2339 MIRBuilder.buildConstant(Res: IntPtrTy, Val: Alignment.value() - 1);
2340 auto ListTmp = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: List, Op1: AlignMinus1.getReg(Idx: 0));
2341 DstPtr = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: ListTmp, NumBits: Log2(A: Alignment));
2342 } else
2343 DstPtr = List;
2344
2345 LLT ValTy = MRI.getType(Reg: Dst);
2346 uint64_t ValSize = ValTy.getSizeInBits() / 8;
2347 MIRBuilder.buildLoad(
2348 Res: Dst, Addr: DstPtr,
2349 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2350 MemTy: ValTy, BaseAlignment: std::max(a: Alignment, b: PtrAlign)));
2351
2352 auto Size = MIRBuilder.buildConstant(Res: IntPtrTy, Val: alignTo(Size: ValSize, A: PtrAlign));
2353
2354 auto NewList = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: DstPtr, Op1: Size.getReg(Idx: 0));
2355
2356 MIRBuilder.buildStore(Val: NewList, Addr: ListPtr,
2357 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
2358 F: MachineMemOperand::MOStore,
2359 MemTy: PtrTy, BaseAlignment: PtrAlign));
2360
2361 MI.eraseFromParent();
2362 return true;
2363}
2364
2365bool AArch64LegalizerInfo::legalizeBitfieldExtract(
2366 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2367 // Only legal if we can select immediate forms.
2368 // TODO: Lower this otherwise.
2369 return getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI) &&
2370 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
2371}
2372
2373bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
2374 MachineRegisterInfo &MRI,
2375 LegalizerHelper &Helper) const {
2376 // When there is no integer popcount instruction (FEAT_CSSC isn't available),
2377 // it can be more efficiently lowered to the following sequence that uses
2378 // AdvSIMD registers/instructions as long as the copies to/from the AdvSIMD
2379 // registers are cheap.
2380 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
2381 // CNT V0.8B, V0.8B // 8xbyte pop-counts
2382 // ADDV B0, V0.8B // sum 8xbyte pop-counts
2383 // UMOV X0, V0.B[0] // copy byte result back to integer reg
2384 //
2385 // For 128 bit vector popcounts, we lower to the following sequence:
2386 // cnt.16b v0, v0 // v8s16, v4s32, v2s64
2387 // uaddlp.8h v0, v0 // v8s16, v4s32, v2s64
2388 // uaddlp.4s v0, v0 // v4s32, v2s64
2389 // uaddlp.2d v0, v0 // v2s64
2390 //
2391 // For 64 bit vector popcounts, we lower to the following sequence:
2392 // cnt.8b v0, v0 // v4s16, v2s32
2393 // uaddlp.4h v0, v0 // v4s16, v2s32
2394 // uaddlp.2s v0, v0 // v2s32
2395
2396 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2397 Register Dst = MI.getOperand(i: 0).getReg();
2398 Register Val = MI.getOperand(i: 1).getReg();
2399 LLT Ty = MRI.getType(Reg: Val);
2400
2401 LLT i64 = LLT::integer(SizeInBits: 64);
2402 LLT i32 = LLT::integer(SizeInBits: 32);
2403 LLT i16 = LLT::integer(SizeInBits: 16);
2404 LLT i8 = LLT::integer(SizeInBits: 8);
2405 unsigned Size = Ty.getSizeInBits();
2406
2407 assert(Ty == MRI.getType(Dst) &&
2408 "Expected src and dst to have the same type!");
2409
2410 if (ST->hasCSSC() && Ty.isScalar() && Size == 128) {
2411
2412 auto Split = MIRBuilder.buildUnmerge(Res: i64, Op: Val);
2413 auto CTPOP1 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 0));
2414 auto CTPOP2 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 1));
2415 auto Add = MIRBuilder.buildAdd(Dst: i64, Src0: CTPOP1, Src1: CTPOP2);
2416
2417 MIRBuilder.buildZExt(Res: Dst, Op: Add);
2418 MI.eraseFromParent();
2419 return true;
2420 }
2421
2422 if (!ST->hasNEON() ||
2423 MI.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
2424 // Use generic lowering when custom lowering is not possible.
2425 return Ty.isScalar() && (Size == 32 || Size == 64) &&
2426 Helper.lowerBitCount(MI) ==
2427 LegalizerHelper::LegalizeResult::Legalized;
2428 }
2429
2430 // Pre-conditioning: widen Val up to the nearest vector type.
2431 // s32,s64,v4s16,v2s32 -> v8i8
2432 // v8s16,v4s32,v2s64 -> v16i8
2433 LLT VTy = Size == 128 ? LLT::fixed_vector(NumElements: 16, ScalarTy: i8) : LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
2434 if (Ty.isScalar()) {
2435 assert((Size == 32 || Size == 64 || Size == 128) && "Expected only 32, 64, or 128 bit scalars!");
2436 if (Size == 32) {
2437 Val = MIRBuilder.buildZExt(Res: i64, Op: Val).getReg(Idx: 0);
2438 }
2439 }
2440 Val = MIRBuilder.buildBitcast(Dst: VTy, Src: Val).getReg(Idx: 0);
2441
2442 // Count bits in each byte-sized lane.
2443 auto CTPOP = MIRBuilder.buildCTPOP(Dst: VTy, Src0: Val);
2444
2445 // Sum across lanes.
2446 if (ST->hasDotProd() && Ty.isVector() && Ty.getNumElements() >= 2 &&
2447 Ty.getScalarSizeInBits() != 16) {
2448 LLT Dt = Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64) ? LLT::fixed_vector(NumElements: 4, ScalarTy: i32) : Ty;
2449 auto Zeros = MIRBuilder.buildConstant(Res: Dt, Val: 0);
2450 auto Ones = MIRBuilder.buildConstant(Res: VTy, Val: 1);
2451 MachineInstrBuilder Sum;
2452
2453 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2454 auto UDOT =
2455 MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2456 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UADDLP, DstOps: {Ty}, SrcOps: {UDOT});
2457 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2458 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2459 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2460 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2461 } else {
2462 llvm_unreachable("unexpected vector shape");
2463 }
2464
2465 Sum->getOperand(i: 0).setReg(Dst);
2466 MI.eraseFromParent();
2467 return true;
2468 }
2469
2470 Register HSum = CTPOP.getReg(Idx: 0);
2471 unsigned Opc;
2472 SmallVector<LLT> HAddTys;
2473 if (Ty.isScalar()) {
2474 Opc = Intrinsic::aarch64_neon_uaddlv;
2475 HAddTys.push_back(Elt: i32);
2476 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: i16)) {
2477 Opc = Intrinsic::aarch64_neon_uaddlp;
2478 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2479 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2480 Opc = Intrinsic::aarch64_neon_uaddlp;
2481 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2482 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2483 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2484 Opc = Intrinsic::aarch64_neon_uaddlp;
2485 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2486 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2487 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i64));
2488 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i16)) {
2489 Opc = Intrinsic::aarch64_neon_uaddlp;
2490 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2491 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2492 Opc = Intrinsic::aarch64_neon_uaddlp;
2493 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2494 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i32));
2495 } else
2496 llvm_unreachable("unexpected vector shape");
2497 MachineInstrBuilder UADD;
2498 for (LLT HTy : HAddTys) {
2499 UADD = MIRBuilder.buildIntrinsic(ID: Opc, Res: {HTy}).addUse(RegNo: HSum);
2500 HSum = UADD.getReg(Idx: 0);
2501 }
2502
2503 // Post-conditioning.
2504 if (Ty.isScalar() && (Size == 64 || Size == 128))
2505 MIRBuilder.buildZExt(Res: Dst, Op: UADD);
2506 else
2507 UADD->getOperand(i: 0).setReg(Dst);
2508 MI.eraseFromParent();
2509 return true;
2510}
2511
2512bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
2513 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2514 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2515 LLT i64 = LLT::integer(SizeInBits: 64);
2516 auto Addr = MI.getOperand(i: 1).getReg();
2517 auto DesiredI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 2));
2518 auto NewI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 3));
2519 auto DstLo = MRI.createGenericVirtualRegister(Ty: i64);
2520 auto DstHi = MRI.createGenericVirtualRegister(Ty: i64);
2521
2522 MachineInstrBuilder CAS;
2523 if (ST->hasLSE()) {
2524 // We have 128-bit CASP instructions taking XSeqPair registers, which are
2525 // s128. We need the merge/unmerge to bracket the expansion and pair up with
2526 // the rest of the MIR so we must reassemble the extracted registers into a
2527 // 128-bit known-regclass one with code like this:
2528 //
2529 // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
2530 // %out = CASP %in1, ...
2531 // %OldLo = G_EXTRACT %out, 0
2532 // %OldHi = G_EXTRACT %out, 64
2533 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2534 unsigned Opcode;
2535 switch (Ordering) {
2536 case AtomicOrdering::Acquire:
2537 Opcode = AArch64::CASPAX;
2538 break;
2539 case AtomicOrdering::Release:
2540 Opcode = AArch64::CASPLX;
2541 break;
2542 case AtomicOrdering::AcquireRelease:
2543 case AtomicOrdering::SequentiallyConsistent:
2544 Opcode = AArch64::CASPALX;
2545 break;
2546 default:
2547 Opcode = AArch64::CASPX;
2548 break;
2549 }
2550
2551 LLT s128 = LLT::integer(SizeInBits: 128);
2552 auto CASDst = MRI.createGenericVirtualRegister(Ty: s128);
2553 auto CASDesired = MRI.createGenericVirtualRegister(Ty: s128);
2554 auto CASNew = MRI.createGenericVirtualRegister(Ty: s128);
2555 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASDesired}, SrcOps: {})
2556 .addUse(RegNo: DesiredI->getOperand(i: 0).getReg())
2557 .addImm(Val: AArch64::sube64)
2558 .addUse(RegNo: DesiredI->getOperand(i: 1).getReg())
2559 .addImm(Val: AArch64::subo64);
2560 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASNew}, SrcOps: {})
2561 .addUse(RegNo: NewI->getOperand(i: 0).getReg())
2562 .addImm(Val: AArch64::sube64)
2563 .addUse(RegNo: NewI->getOperand(i: 1).getReg())
2564 .addImm(Val: AArch64::subo64);
2565
2566 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {CASDst}, SrcOps: {CASDesired, CASNew, Addr});
2567
2568 MIRBuilder.buildExtract(Res: {DstLo}, Src: {CASDst}, Index: 0);
2569 MIRBuilder.buildExtract(Res: {DstHi}, Src: {CASDst}, Index: 64);
2570 } else {
2571 // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
2572 // can take arbitrary registers so it just has the normal GPR64 operands the
2573 // rest of AArch64 is expecting.
2574 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2575 unsigned Opcode;
2576 switch (Ordering) {
2577 case AtomicOrdering::Acquire:
2578 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
2579 break;
2580 case AtomicOrdering::Release:
2581 Opcode = AArch64::CMP_SWAP_128_RELEASE;
2582 break;
2583 case AtomicOrdering::AcquireRelease:
2584 case AtomicOrdering::SequentiallyConsistent:
2585 Opcode = AArch64::CMP_SWAP_128;
2586 break;
2587 default:
2588 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
2589 break;
2590 }
2591
2592 auto Scratch = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2593 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {DstLo, DstHi, Scratch},
2594 SrcOps: {Addr, DesiredI->getOperand(i: 0),
2595 DesiredI->getOperand(i: 1), NewI->getOperand(i: 0),
2596 NewI->getOperand(i: 1)});
2597 }
2598
2599 CAS.cloneMemRefs(OtherMI: MI);
2600 constrainSelectedInstRegOperands(I&: *CAS, TII: *ST->getInstrInfo(),
2601 TRI: *MRI.getTargetRegisterInfo(),
2602 RBI: *ST->getRegBankInfo());
2603
2604 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {DstLo, DstHi});
2605 MI.eraseFromParent();
2606 return true;
2607}
2608
2609bool AArch64LegalizerInfo::legalizeCTTZ(MachineInstr &MI,
2610 LegalizerHelper &Helper) const {
2611 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2612 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2613 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2614 auto BitReverse = MIRBuilder.buildBitReverse(Dst: Ty, Src: MI.getOperand(i: 1));
2615 MIRBuilder.buildCTLZ(Dst: MI.getOperand(i: 0).getReg(), Src0: BitReverse);
2616 MI.eraseFromParent();
2617 return true;
2618}
2619
2620bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
2621 LegalizerHelper &Helper) const {
2622 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2623
2624 // Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
2625 if (MI.getOpcode() == TargetOpcode::G_MEMSET ||
2626 MI.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2627 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
2628 // the instruction).
2629 auto &Value = MI.getOperand(i: 1);
2630 Register ExtValueReg =
2631 MIRBuilder.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
2632 Value.setReg(ExtValueReg);
2633 return true;
2634 }
2635
2636 return false;
2637}
2638
2639bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2640 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2641 const GExtractVectorElement *Element = cast<GExtractVectorElement>(Val: &MI);
2642 auto VRegAndVal =
2643 getIConstantVRegValWithLookThrough(VReg: Element->getIndexReg(), MRI);
2644 if (VRegAndVal)
2645 return true;
2646 LLT VecTy = MRI.getType(Reg: Element->getVectorReg());
2647 if (VecTy.isScalableVector())
2648 return true;
2649 return Helper.lowerExtractInsertVectorElt(MI) !=
2650 LegalizerHelper::LegalizeResult::UnableToLegalize;
2651}
2652
2653bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2654 MachineInstr &MI, LegalizerHelper &Helper) const {
2655 MachineFunction &MF = *MI.getParent()->getParent();
2656 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2657 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2658
2659 // If stack probing is not enabled for this function, use the default
2660 // lowering.
2661 if (!MF.getFunction().hasFnAttribute(Kind: "probe-stack") ||
2662 MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() !=
2663 "inline-asm") {
2664 Helper.lowerDynStackAlloc(MI);
2665 return true;
2666 }
2667
2668 Register Dst = MI.getOperand(i: 0).getReg();
2669 Register AllocSize = MI.getOperand(i: 1).getReg();
2670 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
2671
2672 assert(MRI.getType(Dst) == LLT::pointer(0, 64) &&
2673 "Unexpected type for dynamic alloca");
2674 assert(MRI.getType(AllocSize) == LLT::scalar(64) &&
2675 "Unexpected type for dynamic alloca");
2676
2677 LLT PtrTy = MRI.getType(Reg: Dst);
2678 Register SPReg =
2679 Helper.getTargetLowering().getStackPointerRegisterToSaveRestore();
2680 Register SPTmp =
2681 Helper.getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
2682 auto NewMI =
2683 MIRBuilder.buildInstr(Opc: AArch64::PROBED_STACKALLOC_DYN, DstOps: {}, SrcOps: {SPTmp});
2684 MRI.setRegClass(Reg: NewMI.getReg(Idx: 0), RC: &AArch64::GPR64commonRegClass);
2685 MIRBuilder.setInsertPt(MBB&: *NewMI->getParent(), II: NewMI);
2686 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
2687
2688 MI.eraseFromParent();
2689 return true;
2690}
2691
2692bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
2693 LegalizerHelper &Helper) const {
2694 MachineIRBuilder &MIB = Helper.MIRBuilder;
2695 auto &AddrVal = MI.getOperand(i: 0);
2696
2697 int64_t IsWrite = MI.getOperand(i: 1).getImm();
2698 int64_t Locality = MI.getOperand(i: 2).getImm();
2699 int64_t IsData = MI.getOperand(i: 3).getImm();
2700
2701 bool IsStream = Locality == 0;
2702 if (Locality != 0) {
2703 assert(Locality <= 3 && "Prefetch locality out-of-range");
2704 // The locality degree is the opposite of the cache speed.
2705 // Put the number the other way around.
2706 // The encoding starts at 0 for level 1
2707 Locality = 3 - Locality;
2708 }
2709
2710 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2711
2712 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
2713 MI.eraseFromParent();
2714 return true;
2715}
2716
2717bool AArch64LegalizerInfo::legalizeConcatVectors(
2718 MachineInstr &MI, MachineRegisterInfo &MRI,
2719 MachineIRBuilder &MIRBuilder) const {
2720 // Widen sub-byte element vectors to byte-sized elements before concatenating.
2721 // This is analogous to SDAG's integer type promotion for sub-byte types.
2722 auto &Concat = cast<GConcatVectors>(Val&: MI);
2723 Register DstReg = Concat.getReg(Idx: 0);
2724 LLT DstTy = MRI.getType(Reg: DstReg);
2725 assert(DstTy.getScalarSizeInBits() < 8 && "Expected dst ty to be < 8b");
2726
2727 unsigned WideEltSize =
2728 std::max(a: 8u, b: (unsigned)PowerOf2Ceil(A: DstTy.getScalarSizeInBits()));
2729 LLT SrcTy = MRI.getType(Reg: Concat.getSourceReg(I: 0));
2730 LLT WideSrcTy = SrcTy.changeElementSize(NewEltSize: WideEltSize);
2731 LLT WideDstTy = DstTy.changeElementSize(NewEltSize: WideEltSize);
2732
2733 SmallVector<Register> WideSrcs;
2734 for (unsigned I = 0; I < Concat.getNumSources(); ++I) {
2735 auto Wide = MIRBuilder.buildAnyExt(Res: WideSrcTy, Op: Concat.getSourceReg(I));
2736 WideSrcs.push_back(Elt: Wide.getReg(Idx: 0));
2737 }
2738
2739 auto WideConcat = MIRBuilder.buildConcatVectors(Res: WideDstTy, Ops: WideSrcs);
2740 MIRBuilder.buildTrunc(Res: DstReg, Op: WideConcat);
2741 MI.eraseFromParent();
2742 return true;
2743}
2744
2745bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
2746 MachineIRBuilder &MIRBuilder,
2747 MachineRegisterInfo &MRI) const {
2748 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
2749
2750 // This function legalizes f64 -> bf16 and f64 -> f16 truncations via f64 ->
2751 // f32 G_FPTRUNC_ODD and f32 -> [b]f16 G_FPTRUNC, which apparently avoids the
2752 // usual double-rounding issue that could be present from using twin
2753 // G_FPTRUNC.
2754
2755 if (DstTy.isBFloat16() && SrcTy.isFloat64()) {
2756 auto Mid = MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {LLT::float32()},
2757 SrcOps: {Src}, Flags: MI.getFlags());
2758 MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC, DstOps: {Dst}, SrcOps: {Mid}, Flags: MI.getFlags());
2759 MI.eraseFromParent();
2760 return true;
2761 }
2762
2763 assert(SrcTy.isFixedVector() && isPowerOf2_32(SrcTy.getNumElements()) &&
2764 "Expected a power of 2 elements");
2765
2766 // We must mutate types here as FPTrunc may be used on a IEEE floating point
2767 // or a brainfloat.
2768 LLT v2s16 = DstTy.changeElementCount(NumElements: 2);
2769 LLT v4s16 = DstTy.changeElementCount(NumElements: 4);
2770 LLT v2s32 = SrcTy.changeElementCount(NumElements: 2).changeElementSize(NewEltSize: 32);
2771 LLT v4s32 = SrcTy.changeElementCount(NumElements: 4).changeElementSize(NewEltSize: 32);
2772 LLT v2s64 = SrcTy.changeElementCount(NumElements: 2);
2773
2774 SmallVector<Register> RegsToUnmergeTo;
2775 SmallVector<Register> TruncOddDstRegs;
2776 SmallVector<Register> RegsToMerge;
2777
2778 unsigned ElemCount = SrcTy.getNumElements();
2779
2780 // Find the biggest size chunks we can work with
2781 int StepSize = ElemCount % 4 ? 2 : 4;
2782
2783 // If we have a power of 2 greater than 2, we need to first unmerge into
2784 // enough pieces
2785 if (ElemCount <= 2)
2786 RegsToUnmergeTo.push_back(Elt: Src);
2787 else {
2788 for (unsigned i = 0; i < ElemCount / 2; ++i)
2789 RegsToUnmergeTo.push_back(Elt: MRI.createGenericVirtualRegister(Ty: v2s64));
2790
2791 MIRBuilder.buildUnmerge(Res: RegsToUnmergeTo, Op: Src);
2792 }
2793
2794 // Create all of the round-to-odd instructions and store them
2795 for (auto SrcReg : RegsToUnmergeTo) {
2796 Register Mid = MIRBuilder
2797 .buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {v2s32}, SrcOps: {SrcReg},
2798 Flags: MI.getFlags())
2799 .getReg(Idx: 0);
2800 TruncOddDstRegs.push_back(Elt: Mid);
2801 }
2802
2803 // Truncate 4s32 to 4s16 if we can to reduce instruction count, otherwise
2804 // truncate 2s32 to 2s16.
2805 unsigned Index = 0;
2806 for (unsigned LoopIter = 0; LoopIter < ElemCount / StepSize; ++LoopIter) {
2807 if (StepSize == 4) {
2808 Register ConcatDst =
2809 MIRBuilder
2810 .buildMergeLikeInstr(
2811 Res: {v4s32}, Ops: {TruncOddDstRegs[Index++], TruncOddDstRegs[Index++]})
2812 .getReg(Idx: 0);
2813
2814 RegsToMerge.push_back(
2815 Elt: MIRBuilder.buildFPTrunc(Res: v4s16, Op: ConcatDst, Flags: MI.getFlags()).getReg(Idx: 0));
2816 } else {
2817 RegsToMerge.push_back(
2818 Elt: MIRBuilder
2819 .buildFPTrunc(Res: v2s16, Op: TruncOddDstRegs[Index++], Flags: MI.getFlags())
2820 .getReg(Idx: 0));
2821 }
2822 }
2823
2824 // If there is only one register, replace the destination
2825 if (RegsToMerge.size() == 1) {
2826 MRI.replaceRegWith(FromReg: Dst, ToReg: RegsToMerge.pop_back_val());
2827 MI.eraseFromParent();
2828 return true;
2829 }
2830
2831 // Merge the rest of the instructions & replace the register
2832 Register Fin = MIRBuilder.buildMergeLikeInstr(Res: DstTy, Ops: RegsToMerge).getReg(Idx: 0);
2833 MRI.replaceRegWith(FromReg: Dst, ToReg: Fin);
2834 MI.eraseFromParent();
2835 return true;
2836}
2837
2838bool AArch64LegalizerInfo::legalizeGetRounding(MachineInstr &MI,
2839 MachineIRBuilder &MIRBuilder,
2840 MachineRegisterInfo &MRI,
2841 LegalizerHelper &Helper) const {
2842 const LLT I32 = LLT::integer(SizeInBits: 32);
2843 const LLT I64 = LLT::integer(SizeInBits: 64);
2844
2845 Register Dst = MI.getOperand(i: 0).getReg();
2846 Register FPCR64 = MRI.createGenericVirtualRegister(Ty: I64);
2847 MachineInstrBuilder GetFPCR =
2848 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_get_fpcr, Res: ArrayRef{FPCR64});
2849
2850 // AArch64 rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3,
2851 // 3->0, so we add one to the FPCR bits for the rounding mode.
2852 // Instead of shifting and then adding as `((FPCR >> 22) + 1) & 0b11` which
2853 // generates 3 instructions, we increment the rounding mode with
2854 // `(FPCR + (1 << 22))` and extract the bits. The shift and addition is done
2855 // in one instruction as `add .., .., #1024, lsl #12`, so overall we generate
2856 // one less instruction.
2857 auto FPCR32 = MIRBuilder.buildTrunc(Res: I32, Op: GetFPCR);
2858 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1U << 22);
2859 auto Added = MIRBuilder.buildAdd(Dst: I32, Src0: FPCR32, Src1: One);
2860 auto LSB = MIRBuilder.buildConstant(Res: I32, Val: 22);
2861 auto Width = MIRBuilder.buildConstant(Res: I32, Val: 2);
2862 MIRBuilder.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {Added, LSB, Width});
2863
2864 MI.eraseFromParent();
2865 return true;
2866}
2867
2868bool AArch64LegalizerInfo::legalizeSetRounding(MachineInstr &MI,
2869 MachineIRBuilder &MIRBuilder,
2870 MachineRegisterInfo &MRI,
2871 LegalizerHelper &Helper) const {
2872 const LLT I32 = LLT::integer(SizeInBits: 32);
2873 const LLT I64 = LLT::integer(SizeInBits: 64);
2874
2875 // AArch64 rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3,
2876 // 3->0, so calculate the new value of FPCR[23:22] as `((arg - 1) & 3) << 22`.
2877 Register RM = MI.getOperand(i: 0).getReg();
2878 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
2879 auto Subtracted = MIRBuilder.buildSub(Dst: I32, Src0: RM, Src1: One);
2880 auto Mask = MIRBuilder.buildConstant(Res: I32, Val: 0b11);
2881 auto Masked = MIRBuilder.buildAnd(Dst: I32, Src0: Subtracted, Src1: Mask);
2882 auto ShiftAmount = MIRBuilder.buildConstant(Res: I32, Val: 22);
2883 auto Shifted = MIRBuilder.buildShl(Dst: I32, Src0: Masked, Src1: ShiftAmount);
2884
2885 // Get current value of FPCR.
2886 MachineInstrBuilder GetFPCR =
2887 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_get_fpcr, Res: {I64});
2888
2889 // (FPCR & ~Mask) | Shifted
2890 auto FPCRMask = MIRBuilder.buildConstant(Res: I64, Val: ~((int64_t)0b11 << 22));
2891 auto FPCRMasked = MIRBuilder.buildAnd(Dst: I64, Src0: GetFPCR, Src1: FPCRMask);
2892 auto ShiftedS64 = MIRBuilder.buildZExt(Res: I64, Op: Shifted);
2893 auto FPCRUpdated = MIRBuilder.buildOr(Dst: I64, Src0: FPCRMasked, Src1: ShiftedS64);
2894
2895 // Write new FPCR.
2896 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_set_fpcr, Res: ArrayRef<Register>())
2897 .addUse(RegNo: FPCRUpdated.getReg(Idx: 0));
2898
2899 MI.eraseFromParent();
2900
2901 return true;
2902}
2903