1//===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64LegalizerInfo.h"
15#include "AArch64Subtarget.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetOpcodes.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/IntrinsicsAArch64.h"
30#include "llvm/IR/Type.h"
31#include "llvm/Support/MathExtras.h"
32#include <initializer_list>
33
34#define DEBUG_TYPE "aarch64-legalinfo"
35
36using namespace llvm;
37using namespace LegalizeActions;
38using namespace LegalizeMutations;
39using namespace LegalityPredicates;
40using namespace MIPatternMatch;
41
42AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
43 : ST(&ST) {
44 using namespace TargetOpcode;
45 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
46 const LLT s8 = LLT::scalar(SizeInBits: 8);
47 const LLT s16 = LLT::scalar(SizeInBits: 16);
48 const LLT s32 = LLT::scalar(SizeInBits: 32);
49 const LLT s64 = LLT::scalar(SizeInBits: 64);
50 const LLT s128 = LLT::scalar(SizeInBits: 128);
51 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
52 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
53 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
54 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
55 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
56 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
57 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
58 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
59 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
60 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
61 const LLT v2p0 = LLT::fixed_vector(NumElements: 2, ScalarTy: p0);
62
63 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
64 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
65 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
66 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
67
68 const LLT bf16 = LLT::bfloat16();
69 const LLT v4bf16 = LLT::fixed_vector(NumElements: 4, ScalarTy: bf16);
70 const LLT v8bf16 = LLT::fixed_vector(NumElements: 8, ScalarTy: bf16);
71
72 const LLT f16 = LLT::float16();
73 const LLT v4f16 = LLT::fixed_vector(NumElements: 4, ScalarTy: f16);
74 const LLT v8f16 = LLT::fixed_vector(NumElements: 8, ScalarTy: f16);
75
76 const LLT f32 = LLT::float32();
77 const LLT v2f32 = LLT::fixed_vector(NumElements: 2, ScalarTy: f32);
78 const LLT v4f32 = LLT::fixed_vector(NumElements: 4, ScalarTy: f32);
79
80 const LLT f64 = LLT::float64();
81 const LLT v2f64 = LLT::fixed_vector(NumElements: 2, ScalarTy: f64);
82
83 const LLT f128 = LLT::float128();
84
85 const LLT i8 = LLT::integer(SizeInBits: 8);
86 const LLT v8i8 = LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
87 const LLT v16i8 = LLT::fixed_vector(NumElements: 16, ScalarTy: i8);
88
89 const LLT i16 = LLT::integer(SizeInBits: 16);
90 const LLT v8i16 = LLT::fixed_vector(NumElements: 8, ScalarTy: i16);
91 const LLT v4i16 = LLT::fixed_vector(NumElements: 4, ScalarTy: i16);
92
93 const LLT i32 = LLT::integer(SizeInBits: 32);
94 const LLT v2i32 = LLT::fixed_vector(NumElements: 2, ScalarTy: i32);
95 const LLT v4i32 = LLT::fixed_vector(NumElements: 4, ScalarTy: i32);
96
97 const LLT i64 = LLT::integer(SizeInBits: 64);
98 const LLT v2i64 = LLT::fixed_vector(NumElements: 2, ScalarTy: i64);
99
100 const LLT i128 = LLT::integer(SizeInBits: 128);
101
102 const LLT nxv16i8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: i8);
103 const LLT nxv8i16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: i16);
104 const LLT nxv4i32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: i32);
105 const LLT nxv2i64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: i64);
106
107 std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
108 v16s8, v8s16, v4s32,
109 v2s64, v2p0,
110 /* End 128bit types */
111 /* Begin 64bit types */
112 v8s8, v4s16, v2s32};
113 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
114 SmallVector<LLT, 8> PackedVectorAllTypesVec(PackedVectorAllTypeList);
115 SmallVector<LLT, 8> ScalarAndPtrTypesVec(ScalarAndPtrTypesList);
116
117 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
118
119 // FIXME: support subtargets which have neon/fp-armv8 disabled.
120 if (!ST.hasNEON() || !ST.hasFPARMv8())
121 return;
122
123 // Some instructions only support s16 if the subtarget has full 16-bit FP
124 // support.
125 const bool HasFP16 = ST.hasFullFP16();
126 const bool HasCSSC = ST.hasCSSC();
127 const bool HasRCPC3 = ST.hasRCPC3();
128 const bool HasSVE = ST.hasSVE();
129
130 getActionDefinitionsBuilder(
131 Opcodes: {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
132 .legalFor(Types: {p0, s8, s16, s32, s64, s128})
133 .legalFor(Types: {v2s8, v4s8, v8s8, v16s8, v2s16, v4s16, v8s16, v2s32, v4s32,
134 v2s64, v2p0})
135 .widenScalarToNextPow2(TypeIdx: 0)
136 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
137 .moreElementsToNextPow2(TypeIdx: 0)
138 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
139 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
140 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
141 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
142 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
143 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
144 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
145
146 getActionDefinitionsBuilder(Opcode: G_PHI)
147 .legalFor(Types: {p0, s16, s32, s64})
148 .legalFor(Types: PackedVectorAllTypeList)
149 .widenScalarToNextPow2(TypeIdx: 0)
150 .moreElementsToNextPow2(TypeIdx: 0)
151 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
152 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: s64)
153 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
154 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
155 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
156 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
157 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
158 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8);
159
160 getActionDefinitionsBuilder(Opcode: G_INSERT)
161 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64, p0}), P1: typeInSet(TypeIdx: 1, TypesInit: {s8, s16, s32}),
162 args: smallerThan(TypeIdx0: 1, TypeIdx1: 0)))
163 .widenScalarToNextPow2(TypeIdx: 0)
164 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
165 .widenScalarToNextPow2(TypeIdx: 1)
166 .minScalar(TypeIdx: 1, Ty: s8)
167 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s32}), TypeIdx: 1, Ty: s16)
168 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s64, p0}), TypeIdx: 1, Ty: s32);
169
170 getActionDefinitionsBuilder(Opcode: G_EXTRACT)
171 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64, p0}),
172 P1: typeInSet(TypeIdx: 1, TypesInit: {s32, s64, s128, p0}), args: smallerThan(TypeIdx0: 0, TypeIdx1: 1)))
173 .widenScalarToNextPow2(TypeIdx: 1)
174 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s128)
175 .widenScalarToNextPow2(TypeIdx: 0)
176 .minScalar(TypeIdx: 0, Ty: s16)
177 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s32}), TypeIdx: 0, Ty: s16)
178 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s64, p0}), TypeIdx: 0, Ty: s32)
179 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s128}), TypeIdx: 0, Ty: s64);
180
181 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB, G_AND, G_OR, G_XOR})
182 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
183 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
184 .widenScalarToNextPow2(TypeIdx: 0)
185 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
186 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
187 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
188 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
189 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
190 .minScalarOrEltIf(
191 Predicate: [=](const LegalityQuery &Query) {
192 return Query.Types[0].getNumElements() <= 2;
193 },
194 TypeIdx: 0, Ty: s32)
195 .minScalarOrEltIf(
196 Predicate: [=](const LegalityQuery &Query) {
197 return Query.Types[0].getNumElements() <= 4;
198 },
199 TypeIdx: 0, Ty: s16)
200 .minScalarOrEltIf(
201 Predicate: [=](const LegalityQuery &Query) {
202 return Query.Types[0].getNumElements() <= 16;
203 },
204 TypeIdx: 0, Ty: s8)
205 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
206 .moreElementsToNextPow2(TypeIdx: 0);
207
208 getActionDefinitionsBuilder(Opcode: G_MUL)
209 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
210 .widenScalarToNextPow2(TypeIdx: 0)
211 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
212 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
213 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
214 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
215 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
216 .minScalarOrEltIf(
217 Predicate: [=](const LegalityQuery &Query) {
218 return Query.Types[0].getNumElements() <= 2;
219 },
220 TypeIdx: 0, Ty: s32)
221 .minScalarOrEltIf(
222 Predicate: [=](const LegalityQuery &Query) {
223 return Query.Types[0].getNumElements() <= 4;
224 },
225 TypeIdx: 0, Ty: s16)
226 .minScalarOrEltIf(
227 Predicate: [=](const LegalityQuery &Query) {
228 return Query.Types[0].getNumElements() <= 16;
229 },
230 TypeIdx: 0, Ty: s8)
231 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
232 .moreElementsToNextPow2(TypeIdx: 0);
233
234 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
235 .customIf(Predicate: [=](const LegalityQuery &Query) {
236 const auto &SrcTy = Query.Types[0];
237 const auto &AmtTy = Query.Types[1];
238 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
239 AmtTy.getSizeInBits() == 32;
240 })
241 .legalFor(Types: {
242 {i32, i32},
243 {i32, i64},
244 {i64, i64},
245 {v8i8, v8i8},
246 {v16i8, v16i8},
247 {v4i16, v4i16},
248 {v8i16, v8i16},
249 {v2i32, v2i32},
250 {v4i32, v4i32},
251 {v2i64, v2i64},
252 })
253 .widenScalarToNextPow2(TypeIdx: 1)
254 .widenScalarToNextPow2(TypeIdx: 0)
255 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
256 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
257 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
258 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
259 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
260 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
261 .moreElementsToNextPow2(TypeIdx: 0)
262 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0)
263 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
264 .minScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, LargeTypeIdx: 0)
265 .maxScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, SmallTypeIdx: 0);
266
267 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
268 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
269 .clampScalarOrElt(TypeIdx: 1, MinTy: s64, MaxTy: s64)
270 .clampNumElements(TypeIdx: 0, MinTy: v2p0, MaxTy: v2p0);
271
272 getActionDefinitionsBuilder(Opcode: G_PTRMASK).legalFor(Types: {{p0, s64}});
273
274 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV})
275 .legalFor(Types: {i32, i64})
276 .libcallFor(Types: {i128})
277 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
278 .widenScalarToNextPow2(TypeIdx: 0)
279 .scalarize(TypeIdx: 0);
280
281 getActionDefinitionsBuilder(Opcodes: {G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
282 .lowerFor(Types: {i8, i16, i32, i64, v2i32, v4i32, v2i64})
283 .libcallFor(Types: {i128})
284 .widenScalarOrEltToNextPow2(TypeIdx: 0)
285 .minScalarOrElt(TypeIdx: 0, Ty: s32)
286 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
287 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
288 .scalarize(TypeIdx: 0);
289
290 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
291 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
292 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
293 .lower();
294
295 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
296 .legalFor(Types: {i64, v16i8, v8i16, v4i32})
297 .lower();
298
299 getActionDefinitionsBuilder(
300 Opcodes: {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT})
301 .lower();
302
303 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX})
304 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
305 .legalFor(Pred: HasCSSC, Types: {i32, i64})
306 .minScalar(Pred: HasCSSC, TypeIdx: 0, Ty: s32)
307 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
308 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
309 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
310 .lower();
311
312 // FIXME: Legal vector types are only legal with NEON.
313 getActionDefinitionsBuilder(Opcode: G_ABS)
314 .legalFor(Pred: HasCSSC, Types: {i32, i64})
315 .legalFor(Types: {v16i8, v8i16, v4i32, v2i64, v2p0, v8i8, v4i16, v2i32})
316 .customIf(Predicate: [=](const LegalityQuery &Q) {
317 // TODO: Fix suboptimal codegen for 128+ bit types.
318 LLT SrcTy = Q.Types[0];
319 return SrcTy.isScalar() && SrcTy.getSizeInBits() < 128;
320 })
321 .widenScalarIf(
322 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v4s8; },
323 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v4i16); })
324 .widenScalarIf(
325 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v2s16; },
326 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v2i32); })
327 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
328 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
329 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
330 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
331 .moreElementsToNextPow2(TypeIdx: 0)
332 .lower();
333
334 getActionDefinitionsBuilder(
335 Opcodes: {G_ABDS, G_ABDU, G_UAVGFLOOR, G_UAVGCEIL, G_SAVGFLOOR, G_SAVGCEIL})
336 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
337 .lower();
338
339 getActionDefinitionsBuilder(
340 Opcodes: {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
341 .legalFor(Types: {{i32, i32}, {i64, i32}})
342 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
343 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
344 .widenScalarToNextPow2(TypeIdx: 0);
345
346 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR})
347 .customFor(Types: {{i32, i32}, {i32, i64}, {i64, i64}})
348 .lower();
349
350 getActionDefinitionsBuilder(Opcode: G_ROTR)
351 .legalFor(Types: {{i32, i64}, {i64, i64}})
352 .customIf(Predicate: [=](const LegalityQuery &Q) {
353 return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
354 })
355 .lower();
356 getActionDefinitionsBuilder(Opcode: G_ROTL).lower();
357
358 getActionDefinitionsBuilder(Opcodes: {G_SBFX, G_UBFX})
359 .customFor(Types: {{s32, s32}, {s64, s64}});
360
361 auto always = [=](const LegalityQuery &Q) { return true; };
362 getActionDefinitionsBuilder(Opcode: G_CTPOP)
363 .legalFor(Pred: HasCSSC, Types: {{i32, i32}, {i64, i64}})
364 .legalFor(Types: {{v8i8, v8i8}, {v16i8, v16i8}})
365 .customFor(Pred: !HasCSSC, Types: {{s32, s32}, {s64, s64}})
366 .customFor(Types: {{s128, s128},
367 {v4s16, v4s16},
368 {v8s16, v8s16},
369 {v2s32, v2s32},
370 {v4s32, v4s32},
371 {v2s64, v2s64}})
372 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s128)
373 .widenScalarToNextPow2(TypeIdx: 0)
374 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
375 .minScalarEltSameAsIf(Predicate: always, TypeIdx: 1, LargeTypeIdx: 0)
376 .maxScalarEltSameAsIf(Predicate: always, TypeIdx: 1, SmallTypeIdx: 0)
377 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
378 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
379 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
380 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
381 .moreElementsToNextPow2(TypeIdx: 0)
382 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
383
384 getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTLS})
385 .legalFor(Types: {{i32, i32},
386 {i64, i64},
387 {v8i8, v8i8},
388 {v16i8, v16i8},
389 {v4i16, v4i16},
390 {v8i16, v8i16},
391 {v2i32, v2i32},
392 {v4i32, v4i32}})
393 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
394 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
395 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, /*Min=*/MinSize: 8)
396 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
397 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
398 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
399 .moreElementsToNextPow2(TypeIdx: 0)
400 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 32), TypeIdx: 0)
401 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
402
403 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR).lower();
404
405 getActionDefinitionsBuilder(Opcode: G_CTLZ_ZERO_POISON).lower();
406
407 getActionDefinitionsBuilder(Opcode: G_CTTZ)
408 .lowerIf(Predicate: isVector(TypeIdx: 0))
409 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
410 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
411 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1)
412 .legalFor(Pred: HasCSSC, Types: {s32, s64})
413 .customFor(Pred: !HasCSSC, Types: {s32, s64});
414
415 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON).lower();
416
417 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
418 .legalFor(Types: {i32, i64, v8i8, v16i8})
419 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
420 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
421 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
422 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
423 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
424 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
425 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
426 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
427 .moreElementsToNextPow2(TypeIdx: 0)
428 .lower();
429
430 getActionDefinitionsBuilder(Opcode: G_CLMUL).legalFor(Types: {v8i8, v16i8});
431
432 getActionDefinitionsBuilder(Opcode: G_BSWAP)
433 .legalFor(Types: {i32, i64, v4i16, v8i16, v2i32, v4i32, v2i64})
434 .widenScalarOrEltToNextPow2(TypeIdx: 0, MinSize: 16)
435 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
436 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
437 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
438 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
439 .moreElementsToNextPow2(TypeIdx: 0);
440
441 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT})
442 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
443 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
444 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
445 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
446 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
447 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
448 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
449 .moreElementsToNextPow2(TypeIdx: 0)
450 .lower();
451
452 getActionDefinitionsBuilder(
453 Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
454 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
455 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
456 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
457 .legalFor(Pred: HasFP16, Types: {f16, v4f16, v8f16})
458 .libcallFor(Types: {f128})
459 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
460 .widenScalarIf(
461 Predicate: [=](const LegalityQuery &Q) {
462 return (!HasFP16 && Q.Types[0].getScalarType().isFloat16()) ||
463 Q.Types[0].getScalarType().isBFloat16();
464 },
465 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
466 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
467 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
468 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
469 .moreElementsToNextPow2(TypeIdx: 0);
470
471 getActionDefinitionsBuilder(Opcodes: {G_FABS, G_FNEG})
472 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
473 .legalFor(Pred: HasFP16, Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16})
474 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
475 .lowerIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64))
476 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
477 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
478 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
479 .moreElementsToNextPow2(TypeIdx: 0)
480 .lowerFor(Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16});
481
482 getActionDefinitionsBuilder(Opcodes: {G_FREM, G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
483 G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10,
484 G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH,
485 G_FSINH, G_FTANH, G_FMODF})
486 .libcallFor(Types: {f32, f64, f128})
487 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
488 .scalarize(TypeIdx: 0);
489 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
490 .libcallFor(Types: {{f32, i32}, {f64, i32}, {f128, i32}})
491 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
492 .scalarize(TypeIdx: 0);
493
494 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_INTRINSIC_LRINT})
495 .legalFor(Types: {{i32, f32}, {i32, f64}, {i64, f32}, {i64, f64}})
496 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {i64, f16}})
497 .minScalar(TypeIdx: 1, Ty: s32)
498 .libcallFor(Types: {{s64, s128}})
499 .lower();
500 getActionDefinitionsBuilder(Opcodes: {G_LLROUND, G_INTRINSIC_LLRINT})
501 .legalFor(Types: {{i64, f32}, {i64, f64}})
502 .legalFor(Pred: HasFP16, Types: {{i64, f16}})
503 .minScalar(TypeIdx: 0, Ty: s64)
504 .minScalar(TypeIdx: 1, Ty: s32)
505 .libcallFor(Types: {{s64, s128}})
506 .lower();
507
508 // TODO: Custom legalization for mismatched types.
509 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
510 .moreElementsIf(
511 Predicate: [](const LegalityQuery &Query) { return Query.Types[0].isScalar(); },
512 Mutation: [=](const LegalityQuery &Query) {
513 const LLT Ty = Query.Types[0];
514 return std::pair(0, LLT::fixed_vector(NumElements: Ty == s16 ? 4 : 2, ScalarTy: Ty));
515 })
516 .lower();
517
518 getActionDefinitionsBuilder(Opcode: G_FMAD).lower();
519
520 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
521 auto &Actions = getActionDefinitionsBuilder(Opcode: Op);
522
523 if (Op == G_SEXTLOAD)
524 Actions.lowerIf(Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered));
525
526 // Atomics have zero extending behavior.
527 Actions
528 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8},
529 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8},
530 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
531 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 2},
532 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 2},
533 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 4},
534 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
535 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
536 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}})
537 .widenScalarToNextPow2(TypeIdx: 0)
538 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
539 // TODO: We could support sum-of-pow2's but the lowering code doesn't know
540 // how to do that yet.
541 .unsupportedIfMemSizeNotPow2()
542 // Lower anything left over into G_*EXT and G_LOAD
543 .lower();
544 }
545
546 auto IsPtrVecPred = [=](const LegalityQuery &Query) {
547 const LLT &ValTy = Query.Types[0];
548 return ValTy.isPointerVector() && ValTy.getAddressSpace() == 0;
549 };
550
551 getActionDefinitionsBuilder(Opcode: G_LOAD)
552 .customIf(Predicate: [=](const LegalityQuery &Query) {
553 return HasRCPC3 && Query.Types[0] == s128 &&
554 Query.MMODescrs[0].Ordering == AtomicOrdering::Acquire;
555 })
556 .customIf(Predicate: [=](const LegalityQuery &Query) {
557 return Query.Types[0] == s128 &&
558 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
559 })
560 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8},
561 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8},
562 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
563 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
564 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
565 {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8},
566 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8},
567 {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
568 {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8},
569 {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
570 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8},
571 {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8},
572 {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
573 // These extends are also legal
574 .legalForTypesWithMemDesc(
575 TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}})
576 .legalForTypesWithMemDesc(TypesAndMemDesc: {
577 // SVE vscale x 128 bit base sizes
578 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
579 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
580 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
581 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
582 })
583 .widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
584 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
585 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
586 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
587 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
588 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
589 .lowerIfMemSizeNotByteSizePow2()
590 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
591 .narrowScalarIf(
592 Predicate: [=](const LegalityQuery &Query) {
593 // Clamp extending load results to 32-bits.
594 return Query.Types[0].isScalar() &&
595 Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
596 Query.Types[0].getSizeInBits() > 32;
597 },
598 Mutation: changeTo(TypeIdx: 0, Ty: s32))
599 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
600 .bitcastIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
601 Mutation: [=](const LegalityQuery &Query) {
602 const LLT VecTy = Query.Types[0];
603 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
604 })
605 .customIf(Predicate: IsPtrVecPred)
606 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
607 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
608
609 getActionDefinitionsBuilder(Opcode: G_STORE)
610 .customIf(Predicate: [=](const LegalityQuery &Query) {
611 return HasRCPC3 && Query.Types[0] == s128 &&
612 Query.MMODescrs[0].Ordering == AtomicOrdering::Release;
613 })
614 .customIf(Predicate: [=](const LegalityQuery &Query) {
615 return Query.Types[0] == s128 &&
616 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
617 })
618 .widenScalarIf(
619 Predicate: all(P0: scalarNarrowerThan(TypeIdx: 0, Size: 32),
620 P1: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Release)),
621 Mutation: changeElementSizeTo(TypeIdx: 0, NewTy: s32))
622 .legalForTypesWithMemDesc(
623 TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s16
624 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s32
625 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s64
626 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s32
627 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s64
628 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
629 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}, // truncstorei32 from s64
630 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
631 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
632 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
633 .legalForTypesWithMemDesc(TypesAndMemDesc: {
634 // SVE vscale x 128 bit base sizes
635 // TODO: Add nxv2p0. Consider bitcastIf.
636 // See #92130
637 // https://github.com/llvm/llvm-project/pull/92130#discussion_r1616888461
638 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
639 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
640 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
641 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
642 })
643 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
644 .minScalarOrElt(TypeIdx: 0, Ty: s8)
645 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
646 return Query.Types[0].isScalar() &&
647 Query.Types[0] != Query.MMODescrs[0].MemoryTy;
648 })
649 // Maximum: sN * k = 128
650 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
651 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
652 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
653 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
654 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
655 .lowerIfMemSizeNotPow2()
656 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
657 .bitcastIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
658 P1: LegalityPredicate([=](const LegalityQuery &Query) {
659 return Query.Types[0].getSizeInBits() ==
660 Query.MMODescrs[0].MemoryTy.getSizeInBits();
661 })),
662 Mutation: [=](const LegalityQuery &Query) {
663 const LLT VecTy = Query.Types[0];
664 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
665 })
666 .customIf(Predicate: IsPtrVecPred)
667 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
668 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
669 .lower();
670
671 getActionDefinitionsBuilder(Opcode: G_INDEXED_STORE)
672 // Idx 0 == Ptr, Idx 1 == Val
673 // TODO: we can implement legalizations but as of now these are
674 // generated in a very specific way.
675 .legalForTypesWithMemDesc(TypesAndMemDesc: {
676 {.Type0: p0, .Type1: s8, .MemTy: s8, .Align: 8},
677 {.Type0: p0, .Type1: s16, .MemTy: s16, .Align: 8},
678 {.Type0: p0, .Type1: s32, .MemTy: s8, .Align: 8},
679 {.Type0: p0, .Type1: s32, .MemTy: s16, .Align: 8},
680 {.Type0: p0, .Type1: s32, .MemTy: s32, .Align: 8},
681 {.Type0: p0, .Type1: s64, .MemTy: s64, .Align: 8},
682 {.Type0: p0, .Type1: p0, .MemTy: p0, .Align: 8},
683 {.Type0: p0, .Type1: v8s8, .MemTy: v8s8, .Align: 8},
684 {.Type0: p0, .Type1: v16s8, .MemTy: v16s8, .Align: 8},
685 {.Type0: p0, .Type1: v4s16, .MemTy: v4s16, .Align: 8},
686 {.Type0: p0, .Type1: v8s16, .MemTy: v8s16, .Align: 8},
687 {.Type0: p0, .Type1: v2s32, .MemTy: v2s32, .Align: 8},
688 {.Type0: p0, .Type1: v4s32, .MemTy: v4s32, .Align: 8},
689 {.Type0: p0, .Type1: v2s64, .MemTy: v2s64, .Align: 8},
690 {.Type0: p0, .Type1: v2p0, .MemTy: v2p0, .Align: 8},
691 {.Type0: p0, .Type1: s128, .MemTy: s128, .Align: 8},
692 })
693 .unsupported();
694
695 auto IndexedLoadBasicPred = [=](const LegalityQuery &Query) {
696 LLT LdTy = Query.Types[0];
697 LLT PtrTy = Query.Types[1];
698 if (!llvm::is_contained(Range: PackedVectorAllTypesVec, Element: LdTy) &&
699 !llvm::is_contained(Range: ScalarAndPtrTypesVec, Element: LdTy) && LdTy != s128)
700 return false;
701 if (PtrTy != p0)
702 return false;
703 return true;
704 };
705 getActionDefinitionsBuilder(Opcode: G_INDEXED_LOAD)
706 .unsupportedIf(
707 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
708 .legalIf(Predicate: IndexedLoadBasicPred)
709 .unsupported();
710 getActionDefinitionsBuilder(Opcodes: {G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD})
711 .unsupportedIf(
712 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
713 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64}),
714 P1: LegalityPredicate([=](const LegalityQuery &Q) {
715 LLT LdTy = Q.Types[0];
716 LLT PtrTy = Q.Types[1];
717 LLT MemTy = Q.MMODescrs[0].MemoryTy;
718 if (PtrTy != p0)
719 return false;
720 if (LdTy == s16)
721 return MemTy == s8;
722 if (LdTy == s32)
723 return MemTy == s8 || MemTy == s16;
724 if (LdTy == s64)
725 return MemTy == s8 || MemTy == s16 || MemTy == s32;
726 return false;
727 })))
728 .unsupported();
729
730 // Constants
731 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
732 .legalFor(Types: {p0, s8, s16, s32, s64})
733 .widenScalarToNextPow2(TypeIdx: 0)
734 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64);
735 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
736 .legalFor(Types: {s16, s32, s64, s128});
737
738 // FIXME: fix moreElementsToNextPow2
739 getActionDefinitionsBuilder(Opcode: G_ICMP)
740 .legalFor(Types: {{i32, i32}, {i32, i64}, {i32, p0}})
741 .widenScalarOrEltToNextPow2(TypeIdx: 1)
742 .minScalarOrElt(TypeIdx: 1, Ty: s8)
743 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
744 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
745 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
746 .minScalarEltSameAsIf(
747 Predicate: [=](const LegalityQuery &Query) {
748 const LLT &Ty = Query.Types[0];
749 const LLT &SrcTy = Query.Types[1];
750 return Ty.isVector() && !SrcTy.isPointerVector() &&
751 Ty.getElementType() != SrcTy.getElementType();
752 },
753 TypeIdx: 0, LargeTypeIdx: 1)
754 .minScalarOrEltIf(
755 Predicate: [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
756 TypeIdx: 1, Ty: s32)
757 .minScalarOrEltIf(
758 Predicate: [=](const LegalityQuery &Query) {
759 return Query.Types[1].isPointerVector();
760 },
761 TypeIdx: 0, Ty: s64)
762 .moreElementsToNextPow2(TypeIdx: 1)
763 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
764 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
765 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
766 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
767 .clampNumElements(TypeIdx: 1, MinTy: v2p0, MaxTy: v2p0)
768 .customIf(Predicate: isVector(TypeIdx: 0));
769
770 getActionDefinitionsBuilder(Opcode: G_FCMP)
771 .legalFor(Types: {{i32, f32},
772 {i32, f64},
773 {v4i32, v4f32},
774 {v2i32, v2f32},
775 {v2i64, v2f64}})
776 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
777 .widenScalarOrEltToNextPow2(TypeIdx: 1)
778 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
779 .widenScalarIf(
780 Predicate: [=](const LegalityQuery &Q) {
781 return (!HasFP16 && Q.Types[1].getScalarType().isFloat16()) ||
782 Q.Types[1].getScalarType().isBFloat16();
783 },
784 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
785 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
786 .minScalarEltSameAsIf(
787 Predicate: [=](const LegalityQuery &Query) {
788 const LLT &Ty = Query.Types[0];
789 const LLT &SrcTy = Query.Types[1];
790 return Ty.isVector() && !SrcTy.isPointerVector() &&
791 Ty.getElementType() != SrcTy.getElementType();
792 },
793 TypeIdx: 0, LargeTypeIdx: 1)
794 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
795 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
796 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
797 .moreElementsToNextPow2(TypeIdx: 1)
798 .libcallFor(Types: {{s32, s128}});
799
800 // Extensions
801 auto ExtLegalFunc = [=](const LegalityQuery &Query) {
802 unsigned DstSize = Query.Types[0].getSizeInBits();
803
804 // Handle legal vectors using legalFor
805 if (Query.Types[0].isVector())
806 return false;
807
808 if (DstSize < 8 || DstSize >= 128 || !isPowerOf2_32(Value: DstSize))
809 return false; // Extending to a scalar s128 needs narrowing.
810
811 const LLT &SrcTy = Query.Types[1];
812
813 // Make sure we fit in a register otherwise. Don't bother checking that
814 // the source type is below 128 bits. We shouldn't be allowing anything
815 // through which is wider than the destination in the first place.
816 unsigned SrcSize = SrcTy.getSizeInBits();
817 if (SrcSize < 8 || !isPowerOf2_32(Value: SrcSize))
818 return false;
819
820 return true;
821 };
822 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
823 .legalIf(Predicate: ExtLegalFunc)
824 .legalFor(Types: {{v8s16, v8s8}, {v4s32, v4s16}, {v2s64, v2s32}})
825 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64) // Just for s128, others are handled above.
826 .moreElementsToNextPow2(TypeIdx: 0)
827 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
828 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
829 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
830 // Tries to convert a large EXTEND into two smaller EXTENDs
831 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
832 return (Query.Types[0].getScalarSizeInBits() >
833 Query.Types[1].getScalarSizeInBits() * 2) &&
834 Query.Types[0].isVector() &&
835 (Query.Types[1].getScalarSizeInBits() == 8 ||
836 Query.Types[1].getScalarSizeInBits() == 16);
837 })
838 .clampMinNumElements(TypeIdx: 1, EltTy: s8, MinElements: 8)
839 .clampMinNumElements(TypeIdx: 1, EltTy: s16, MinElements: 4)
840 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
841
842 getActionDefinitionsBuilder(Opcode: G_TRUNC)
843 .legalFor(Types: {{v8s8, v8s16}, {v4s16, v4s32}, {v2s32, v2s64}})
844 .moreElementsToNextPow2(TypeIdx: 0)
845 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 8)
846 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 4)
847 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 2)
848 .minScalarOrEltIf(
849 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
850 TypeIdx: 0, Ty: s8)
851 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
852 LLT DstTy = Query.Types[0];
853 LLT SrcTy = Query.Types[1];
854 return DstTy.isVector() && SrcTy.getSizeInBits() > 128 &&
855 DstTy.getScalarSizeInBits() * 2 <= SrcTy.getScalarSizeInBits();
856 })
857 .clampMinNumElements(TypeIdx: 0, EltTy: s8, MinElements: 8)
858 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 4)
859 .alwaysLegal();
860
861 getActionDefinitionsBuilder(Opcodes: {G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
862 .legalFor(Types: {{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
863 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v8s8)
864 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v4s16)
865 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v2s32)
866 .lower();
867
868 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
869 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
870 .maxScalar(TypeIdx: 0, Ty: s64)
871 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
872 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
873 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
874 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
875 .lower();
876
877 // FP conversions
878 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
879 .legalFor(
880 Types: {{f16, f32}, {f16, f64}, {f32, f64}, {v4f16, v4f32}, {v2f32, v2f64}})
881 .legalFor(Pred: ST.hasBF16(), Types: {{bf16, f32}, {v4bf16, v4f32}})
882 .libcallFor(Types: {{f16, f128}, {f32, f128}, {f64, f128}})
883 .moreElementsToNextPow2(TypeIdx: 1)
884 .customIf(Predicate: [](const LegalityQuery &Q) {
885 LLT DstTy = Q.Types[0];
886 LLT SrcTy = Q.Types[1];
887 return SrcTy.getScalarSizeInBits() == 64 &&
888 DstTy.getScalarSizeInBits() == 16;
889 })
890 .lowerFor(Types: {{bf16, f32}, {v4bf16, v4f32}})
891 // Clamp based on input
892 .clampNumElements(TypeIdx: 1, MinTy: v4s32, MaxTy: v4s32)
893 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
894 .scalarize(TypeIdx: 0);
895
896 getActionDefinitionsBuilder(Opcode: G_FPEXT)
897 .legalFor(Types: {{f32, f16},
898 {f64, f16},
899 {f32, bf16},
900 {f64, f32},
901 {v4f32, v4f16},
902 {v4f32, v4bf16},
903 {v2f64, v2f32}})
904 .libcallFor(Types: {{f128, f64}, {f128, f32}, {f128, f16}})
905 .moreElementsToNextPow2(TypeIdx: 0)
906 .widenScalarIf(
907 Predicate: [](const LegalityQuery &Q) {
908 LLT DstTy = Q.Types[0];
909 LLT SrcTy = Q.Types[1];
910 return SrcTy.isVector() && DstTy.isVector() &&
911 SrcTy.getScalarSizeInBits() == 16 &&
912 DstTy.getScalarSizeInBits() == 64;
913 },
914 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
915 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
916 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
917 .scalarize(TypeIdx: 0);
918
919 // Conversions
920 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
921 .legalFor(Types: {{i32, f32},
922 {i64, f32},
923 {i32, f64},
924 {i64, f64},
925 {v2i32, v2f32},
926 {v4i32, v4f32},
927 {v2i64, v2f64}})
928 .legalFor(Pred: HasFP16,
929 Types: {{i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
930 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
931 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
932 // The range of a fp16 value fits into an i17, so we can lower the width
933 // to i64.
934 .narrowScalarIf(
935 Predicate: [=](const LegalityQuery &Query) {
936 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
937 },
938 Mutation: changeTo(TypeIdx: 0, Ty: i64))
939 .moreElementsToNextPow2(TypeIdx: 0)
940 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0)
941 .minScalar(TypeIdx: 0, Ty: s32)
942 .widenScalarIf(
943 Predicate: [HasFP16](const LegalityQuery &Query) {
944 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
945 Query.Types[1].getScalarType().isBFloat16();
946 },
947 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
948 .widenScalarIf(
949 Predicate: [=](const LegalityQuery &Query) {
950 return Query.Types[0].getScalarSizeInBits() <= 64 &&
951 Query.Types[0].getScalarSizeInBits() >
952 Query.Types[1].getScalarSizeInBits();
953 },
954 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
955 .widenScalarIf(
956 Predicate: [=](const LegalityQuery &Query) {
957 return Query.Types[1].getScalarSizeInBits() <= 64 &&
958 Query.Types[0].getScalarSizeInBits() <
959 Query.Types[1].getScalarSizeInBits();
960 },
961 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
962 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
963 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
964 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
965 .libcallFor(
966 Types: {{i32, f128}, {i64, f128}, {i128, f128}, {i128, f32}, {i128, f64}});
967
968 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
969 .legalFor(Types: {{i32, f32},
970 {i64, f32},
971 {i32, f64},
972 {i64, f64},
973 {v2i32, v2f32},
974 {v4i32, v4f32},
975 {v2i64, v2f64}})
976 .legalFor(
977 Pred: HasFP16,
978 Types: {{i16, f16}, {i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
979 // Handle types larger than i64 by scalarizing/lowering.
980 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
981 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
982 // The range of a fp16 value fits into an i17, so we can lower the width
983 // to i64.
984 .narrowScalarIf(
985 Predicate: [=](const LegalityQuery &Query) {
986 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
987 },
988 Mutation: changeTo(TypeIdx: 0, Ty: i64))
989 .lowerIf(Predicate: ::any(P0: scalarWiderThan(TypeIdx: 0, Size: 64), P1: scalarWiderThan(TypeIdx: 1, Size: 64)), Mutation: 0)
990 .moreElementsToNextPow2(TypeIdx: 0)
991 .widenScalarToNextPow2(TypeIdx: 0, /*MinSize=*/32)
992 .minScalar(TypeIdx: 0, Ty: s32)
993 .widenScalarIf(
994 Predicate: [HasFP16](const LegalityQuery &Query) {
995 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
996 Query.Types[1].getScalarType().isBFloat16();
997 },
998 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
999 .widenScalarIf(
1000 Predicate: [=](const LegalityQuery &Query) {
1001 unsigned ITySize = Query.Types[0].getScalarSizeInBits();
1002 return (ITySize == 16 || ITySize == 32 || ITySize == 64) &&
1003 ITySize > Query.Types[1].getScalarSizeInBits();
1004 },
1005 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1006 .widenScalarIf(
1007 Predicate: [=](const LegalityQuery &Query) {
1008 unsigned FTySize = Query.Types[1].getScalarSizeInBits();
1009 return (FTySize == 16 || FTySize == 32 || FTySize == 64) &&
1010 Query.Types[0].getScalarSizeInBits() < FTySize;
1011 },
1012 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1013 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1014 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1015 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1016 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1017
1018 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
1019 .legalFor(Types: {{f32, i32},
1020 {f64, i32},
1021 {f32, i64},
1022 {f64, i64},
1023 {v2f32, v2i32},
1024 {v4f32, v4i32},
1025 {v2f64, v2i64}})
1026 .legalFor(Pred: HasFP16,
1027 Types: {{f16, i32}, {f16, i64}, {v4f16, v4i16}, {v8f16, v8i16}})
1028 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1029 return Query.Types[0].getScalarType().isBFloat16();
1030 })
1031 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
1032 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1033 .moreElementsToNextPow2(TypeIdx: 1)
1034 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1)
1035 .minScalar(TypeIdx: 1, Ty: f32)
1036 .lowerIf(Predicate: [](const LegalityQuery &Query) {
1037 return Query.Types[1].isVector() &&
1038 Query.Types[1].getScalarSizeInBits() == 64 &&
1039 Query.Types[0].getScalarSizeInBits() == 16;
1040 })
1041 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, /*MinSize=*/HasFP16 ? 16 : 32)
1042 .scalarizeIf(
1043 // v2i64->v2f32 needs to scalarize to avoid double-rounding issues.
1044 Predicate: [](const LegalityQuery &Query) {
1045 return Query.Types[0].getScalarSizeInBits() == 32 &&
1046 Query.Types[1].getScalarSizeInBits() == 64;
1047 },
1048 TypeIdx: 0)
1049 .widenScalarIf(
1050 Predicate: [](const LegalityQuery &Query) {
1051 return Query.Types[1].getScalarSizeInBits() <= 64 &&
1052 Query.Types[0].getScalarSizeInBits() <
1053 Query.Types[1].getScalarSizeInBits();
1054 },
1055 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1056 .widenScalarIf(
1057 Predicate: [](const LegalityQuery &Query) {
1058 return Query.Types[0].getScalarSizeInBits() <= 64 &&
1059 Query.Types[0].getScalarSizeInBits() >
1060 Query.Types[1].getScalarSizeInBits();
1061 },
1062 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1063 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1064 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1065 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1066 .libcallFor(Types: {{f16, i128},
1067 {f32, i128},
1068 {f64, i128},
1069 {f128, i128},
1070 {f128, i32},
1071 {f128, i64}});
1072
1073 // Control-flow
1074 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
1075 getActionDefinitionsBuilder(Opcode: G_BRCOND)
1076 .legalFor(Types: {s32})
1077 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32);
1078 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
1079
1080 getActionDefinitionsBuilder(Opcode: G_SELECT)
1081 .legalFor(Types: {{s32, s32}, {s64, s32}, {p0, s32}})
1082 .widenScalarToNextPow2(TypeIdx: 0)
1083 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
1084 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s32)
1085 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1086 .minScalarEltSameAsIf(Predicate: all(P0: isVector(TypeIdx: 0), P1: isVector(TypeIdx: 1)), TypeIdx: 1, LargeTypeIdx: 0)
1087 .lowerIf(Predicate: isVector(TypeIdx: 0));
1088
1089 // Pointer-handling
1090 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
1091
1092 if (TM.getCodeModel() == CodeModel::Small)
1093 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).custom();
1094 else
1095 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: {p0});
1096
1097 getActionDefinitionsBuilder(Opcode: G_PTRAUTH_GLOBAL_VALUE)
1098 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: p0), P1: typeIs(TypeIdx: 1, TypesInit: p0)));
1099
1100 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
1101 .legalFor(Types: {{i64, p0}, {v2i64, v2p0}})
1102 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 64)
1103 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64)
1104 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1105
1106 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
1107 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1108 return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
1109 })
1110 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
1111 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2);
1112
1113 // Casts for 32 and 64-bit width type are just copies.
1114 // Same for 128-bit width type, except they are on the FPR bank.
1115 getActionDefinitionsBuilder(Opcode: G_BITCAST)
1116 .legalForCartesianProduct(Types: {s16})
1117 // Keeping 32-bit instructions legal to prevent regression in some tests
1118 .legalForCartesianProduct(Types: {s32, v2s16, v4s8})
1119 .legalForCartesianProduct(Types: {s64, v8s8, v4s16, v2s32})
1120 .legalForCartesianProduct(Types: {s128, v16s8, v8s16, v4s32, v2s64, v2p0})
1121 .customIf(Predicate: [=](const LegalityQuery &Query) {
1122 // Handle casts from i1 vectors to scalars.
1123 LLT DstTy = Query.Types[0];
1124 LLT SrcTy = Query.Types[1];
1125 return DstTy.isScalar() && SrcTy.isVector() &&
1126 SrcTy.getScalarSizeInBits() == 1;
1127 })
1128 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
1129 return Query.Types[0].isVector() != Query.Types[1].isVector();
1130 })
1131 .moreElementsToNextPow2(TypeIdx: 0)
1132 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1133 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1134 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1135 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1136 .lower();
1137
1138 getActionDefinitionsBuilder(Opcode: G_VASTART).legalFor(Types: {p0});
1139
1140 // va_list must be a pointer, but most sized types are pretty easy to handle
1141 // as the destination.
1142 getActionDefinitionsBuilder(Opcode: G_VAARG)
1143 .customForCartesianProduct(Types0: {s8, s16, s32, s64, p0}, Types1: {p0})
1144 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
1145 .widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8);
1146
1147 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
1148 .lowerIf(
1149 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64, s128}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
1150
1151 bool UseOutlineAtomics = ST.outlineAtomics() && !ST.hasLSE();
1152
1153 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG)
1154 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1155 .customFor(Pred: !UseOutlineAtomics, Types: {{s128, p0}})
1156 .libcallFor(Pred: UseOutlineAtomics,
1157 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {s128, p0}})
1158 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1159
1160 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD,
1161 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
1162 G_ATOMICRMW_XOR})
1163 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1164 .libcallFor(Pred: UseOutlineAtomics,
1165 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
1166 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1167
1168 // Do not outline these atomics operations, as per comment in
1169 // AArch64ISelLowering.cpp's shouldExpandAtomicRMWInIR().
1170 getActionDefinitionsBuilder(
1171 Opcodes: {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
1172 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64}), P1: typeIs(TypeIdx: 1, TypesInit: p0)))
1173 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1174
1175 getActionDefinitionsBuilder(Opcode: G_BLOCK_ADDR).legalFor(Types: {p0});
1176
1177 // Merge/Unmerge
1178 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
1179 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
1180 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
1181 getActionDefinitionsBuilder(Opcode: Op)
1182 .widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: 8)
1183 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: 32)
1184 .clampScalar(TypeIdx: LitTyIdx, MinTy: s8, MaxTy: s64)
1185 .clampScalar(TypeIdx: BigTyIdx, MinTy: s32, MaxTy: s128)
1186 .legalIf(Predicate: [=](const LegalityQuery &Q) {
1187 switch (Q.Types[BigTyIdx].getSizeInBits()) {
1188 case 32:
1189 case 64:
1190 case 128:
1191 break;
1192 default:
1193 return false;
1194 }
1195 switch (Q.Types[LitTyIdx].getSizeInBits()) {
1196 case 8:
1197 case 16:
1198 case 32:
1199 case 64:
1200 return true;
1201 default:
1202 return false;
1203 }
1204 });
1205 }
1206
1207 // TODO : nxv4s16, nxv2s16, nxv2s32
1208 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
1209 .legalFor(Pred: HasSVE, Types: {{s16, nxv16s8, s64},
1210 {s16, nxv8s16, s64},
1211 {s32, nxv4s32, s64},
1212 {s64, nxv2s64, s64}})
1213 .unsupportedIf(Predicate: [=](const LegalityQuery &Query) {
1214 const LLT &EltTy = Query.Types[1].getElementType();
1215 if (Query.Types[1].isScalableVector())
1216 return false;
1217 return Query.Types[0] != EltTy;
1218 })
1219 .minScalar(TypeIdx: 2, Ty: s64)
1220 .customIf(Predicate: [=](const LegalityQuery &Query) {
1221 const LLT &VecTy = Query.Types[1];
1222 return VecTy == v8s8 || VecTy == v16s8 || VecTy == v2s16 ||
1223 VecTy == v4s16 || VecTy == v8s16 || VecTy == v2s32 ||
1224 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2p0;
1225 })
1226 .minScalarOrEltIf(
1227 Predicate: [=](const LegalityQuery &Query) {
1228 // We want to promote to <M x s1> to <M x s64> if that wouldn't
1229 // cause the total vec size to be > 128b.
1230 return Query.Types[1].isFixedVector() &&
1231 Query.Types[1].getNumElements() <= 2;
1232 },
1233 TypeIdx: 0, Ty: s64)
1234 .minScalarOrEltIf(
1235 Predicate: [=](const LegalityQuery &Query) {
1236 return Query.Types[1].isFixedVector() &&
1237 Query.Types[1].getNumElements() <= 4;
1238 },
1239 TypeIdx: 0, Ty: s32)
1240 .minScalarOrEltIf(
1241 Predicate: [=](const LegalityQuery &Query) {
1242 return Query.Types[1].isFixedVector() &&
1243 Query.Types[1].getNumElements() <= 8;
1244 },
1245 TypeIdx: 0, Ty: s16)
1246 .minScalarOrEltIf(
1247 Predicate: [=](const LegalityQuery &Query) {
1248 return Query.Types[1].isFixedVector() &&
1249 Query.Types[1].getNumElements() <= 16;
1250 },
1251 TypeIdx: 0, Ty: s8)
1252 .minScalarOrElt(TypeIdx: 0, Ty: s8) // Worst case, we need at least s8.
1253 .moreElementsToNextPow2(TypeIdx: 1)
1254 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1255 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1256 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1257 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1258 .clampMaxNumElements(TypeIdx: 1, EltTy: p0, MaxElements: 2)
1259 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1);
1260
1261 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
1262 .legalIf(
1263 Predicate: typeInSet(TypeIdx: 0, TypesInit: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64, v2p0}))
1264 .legalFor(Pred: HasSVE, Types: {{nxv16s8, s32, s64},
1265 {nxv8s16, s32, s64},
1266 {nxv4s32, s32, s64},
1267 {nxv2s64, s64, s64}})
1268 .moreElementsToNextPow2(TypeIdx: 0)
1269 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1270 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1271 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1272 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1273 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1274 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
1275 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
1276
1277 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
1278 .legalFor(Types: {{v8s8, s8},
1279 {v16s8, s8},
1280 {v4s16, s16},
1281 {v8s16, s16},
1282 {v2s32, s32},
1283 {v4s32, s32},
1284 {v2s64, s64},
1285 {v2p0, p0}})
1286 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1287 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1288 .minScalarOrElt(TypeIdx: 0, Ty: s8)
1289 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1290 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1291 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0);
1292
1293 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR_TRUNC).lower();
1294
1295 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
1296 .legalIf(Predicate: [=](const LegalityQuery &Query) {
1297 const LLT &DstTy = Query.Types[0];
1298 const LLT &SrcTy = Query.Types[1];
1299 // For now just support the TBL2 variant which needs the source vectors
1300 // to be the same size as the dest.
1301 if (DstTy != SrcTy)
1302 return false;
1303 return llvm::is_contained(
1304 Set: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64}, Element: DstTy);
1305 })
1306 .moreElementsIf(
1307 Predicate: [](const LegalityQuery &Query) {
1308 return Query.Types[0].getNumElements() >
1309 Query.Types[1].getNumElements();
1310 },
1311 Mutation: changeTo(TypeIdx: 1, FromTypeIdx: 0))
1312 .moreElementsToNextPow2(TypeIdx: 0)
1313 .moreElementsIf(
1314 Predicate: [](const LegalityQuery &Query) {
1315 return Query.Types[0].getNumElements() <
1316 Query.Types[1].getNumElements();
1317 },
1318 Mutation: changeTo(TypeIdx: 0, FromTypeIdx: 1))
1319 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
1320 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1321 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1322 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1323 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1324 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1325 .bitcastIf(Predicate: isPointerVector(TypeIdx: 0), Mutation: [=](const LegalityQuery &Query) {
1326 // Bitcast pointers vector to i64.
1327 const LLT DstTy = Query.Types[0];
1328 return std::pair(
1329 0, LLT::vector(EC: DstTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 64)));
1330 });
1331
1332 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
1333 .legalFor(Types: {{v16s8, v8s8}, {v8s16, v4s16}, {v4s32, v2s32}})
1334 .customIf(Predicate: [=](const LegalityQuery &Query) {
1335 return Query.Types[0].isFixedVector() &&
1336 Query.Types[0].getScalarSizeInBits() < 8;
1337 })
1338 .bitcastIf(
1339 Predicate: [=](const LegalityQuery &Query) {
1340 return Query.Types[0].isFixedVector() &&
1341 Query.Types[1].isFixedVector() &&
1342 Query.Types[0].getScalarSizeInBits() >= 8 &&
1343 isPowerOf2_64(Value: Query.Types[0].getScalarSizeInBits()) &&
1344 Query.Types[0].getSizeInBits() <= 128 &&
1345 Query.Types[1].getSizeInBits() <= 64;
1346 },
1347 Mutation: [=](const LegalityQuery &Query) {
1348 const LLT DstTy = Query.Types[0];
1349 const LLT SrcTy = Query.Types[1];
1350 return std::pair(
1351 0, DstTy.changeElementSize(NewEltSize: SrcTy.getSizeInBits())
1352 .changeElementCount(
1353 EC: DstTy.getElementCount().divideCoefficientBy(
1354 RHS: SrcTy.getNumElements())));
1355 });
1356
1357 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
1358 .legalFor(Types: {{v8s8, v16s8}, {v4s16, v8s16}, {v2s32, v4s32}})
1359 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1360 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
1361 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
1362 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
1363 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
1364 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
1365 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
1366 .lower()
1367 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1368
1369 // TODO: {nxv16s8, s8}, {nxv8s16, s16}
1370 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
1371 .legalFor(Pred: HasSVE, Types: {{nxv4s32, s32}, {nxv2s64, s64}});
1372
1373 getActionDefinitionsBuilder(Opcode: G_JUMP_TABLE).legalFor(Types: {p0});
1374
1375 getActionDefinitionsBuilder(Opcode: G_BRJT).legalFor(Types: {{p0, s64}});
1376
1377 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
1378
1379 getActionDefinitionsBuilder(Opcode: G_DYN_STACKALLOC).custom();
1380
1381 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).lower();
1382
1383 if (ST.hasMOPS()) {
1384 // G_BZERO is not supported. Currently it is only emitted by
1385 // PreLegalizerCombiner for G_MEMSET with zero constant.
1386 getActionDefinitionsBuilder(Opcode: G_BZERO).unsupported();
1387
1388 getActionDefinitionsBuilder(Opcode: G_MEMSET)
1389 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1390 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64})
1391 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1392
1393 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE})
1394 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64})
1395 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1396
1397 // G_MEMCPY_INLINE does not have a tailcall immediate
1398 getActionDefinitionsBuilder(Opcode: G_MEMCPY_INLINE)
1399 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64});
1400
1401 getActionDefinitionsBuilder(Opcode: G_MEMSET_INLINE)
1402 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1403 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64});
1404 } else {
1405 getActionDefinitionsBuilder(Opcodes: {G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
1406 .libcall();
1407 }
1408
1409 // For fadd reductions we have pairwise operations available. We treat the
1410 // usual legal types as legal and handle the lowering to pairwise instructions
1411 // later.
1412 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FADD)
1413 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1414 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1415 .widenScalarIf(
1416 Predicate: [HasFP16](const LegalityQuery &Query) {
1417 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1418 Query.Types[0].getScalarType().isBFloat16();
1419 },
1420 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1421 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1422 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1423 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1424 .moreElementsToNextPow2(TypeIdx: 1)
1425 .scalarize(TypeIdx: 1)
1426 .lower();
1427
1428 // For fmul reductions we need to split up into individual operations. We
1429 // clamp to 128 bit vectors then to 64bit vectors to produce a cascade of
1430 // smaller types, followed by scalarizing what remains.
1431 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FMUL)
1432 .widenScalarIf(
1433 Predicate: [HasFP16](const LegalityQuery &Query) {
1434 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1435 Query.Types[0].getScalarType().isBFloat16();
1436 },
1437 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1438 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1439 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1440 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1441 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1442 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1443 .scalarize(TypeIdx: 1)
1444 .lower();
1445
1446 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
1447 .scalarize(TypeIdx: 2)
1448 .lower();
1449
1450 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_ADD)
1451 .legalFor(Types: {{i8, v8i8},
1452 {i8, v16i8},
1453 {i16, v4i16},
1454 {i16, v8i16},
1455 {i32, v2i32},
1456 {i32, v4i32},
1457 {i64, v2i64}})
1458 .moreElementsToNextPow2(TypeIdx: 1)
1459 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1460 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1461 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1462 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1463 .widenVectorEltsToVectorMinSize(TypeIdx: 1, VectorSize: 64)
1464 .scalarize(TypeIdx: 1);
1465
1466 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
1467 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1468 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1469 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1470 .widenScalarIf(
1471 Predicate: [HasFP16](const LegalityQuery &Query) {
1472 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1473 Query.Types[0].getScalarType().isBFloat16();
1474 },
1475 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1476 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1477 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1478 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1479 .scalarize(TypeIdx: 1)
1480 .lower();
1481
1482 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_MUL)
1483 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1484 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1485 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
1486 .scalarize(TypeIdx: 1)
1487 .lower();
1488
1489 getActionDefinitionsBuilder(
1490 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1491 .legalFor(Types: {{i8, v8i8},
1492 {i8, v16i8},
1493 {i16, v4i16},
1494 {i16, v8i16},
1495 {i32, v2i32},
1496 {i32, v4i32}})
1497 .moreElementsIf(
1498 Predicate: [=](const LegalityQuery &Query) {
1499 return Query.Types[1].isVector() &&
1500 Query.Types[1].getElementType() != s8 &&
1501 Query.Types[1].getNumElements() & 1;
1502 },
1503 Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx: 1))
1504 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1505 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1506 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1507 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1508 .scalarize(TypeIdx: 1)
1509 .lower();
1510
1511 getActionDefinitionsBuilder(
1512 Opcodes: {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1513 // Try to break down into smaller vectors as long as they're at least 64
1514 // bits. This lets us use vector operations for some parts of the
1515 // reduction.
1516 .fewerElementsIf(
1517 Predicate: [=](const LegalityQuery &Q) {
1518 LLT SrcTy = Q.Types[1];
1519 if (SrcTy.isScalar())
1520 return false;
1521 if (!isPowerOf2_32(Value: SrcTy.getNumElements()))
1522 return false;
1523 // We can usually perform 64b vector operations.
1524 return SrcTy.getSizeInBits() > 64;
1525 },
1526 Mutation: [=](const LegalityQuery &Q) {
1527 LLT SrcTy = Q.Types[1];
1528 return std::make_pair(x: 1, y: SrcTy.divide(Factor: 2));
1529 })
1530 .scalarize(TypeIdx: 1)
1531 .lower();
1532
1533 // TODO: Update this to correct handling when adding AArch64/SVE support.
1534 getActionDefinitionsBuilder(Opcode: G_VECTOR_COMPRESS).lower();
1535
1536 // Access to floating-point environment.
1537 getActionDefinitionsBuilder(Opcodes: {G_GET_FPENV, G_SET_FPENV, G_RESET_FPENV,
1538 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1539 .libcall();
1540
1541 getActionDefinitionsBuilder(Opcodes: {G_GET_ROUNDING, G_SET_ROUNDING})
1542 .customFor(Types: {s32});
1543
1544 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).lower();
1545
1546 getActionDefinitionsBuilder(Opcode: G_PREFETCH).custom();
1547
1548 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
1549
1550 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
1551 .alwaysLegal();
1552 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
1553 getActionDefinitionsBuilder(Opcode: G_INVOKE_REGION_START).alwaysLegal();
1554
1555 verify(MII: *ST.getInstrInfo());
1556}
1557
1558bool AArch64LegalizerInfo::legalizeCustom(
1559 LegalizerHelper &Helper, MachineInstr &MI,
1560 LostDebugLocObserver &LocObserver) const {
1561 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1562 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1563 GISelChangeObserver &Observer = Helper.Observer;
1564 switch (MI.getOpcode()) {
1565 default:
1566 // No idea what to do.
1567 return false;
1568 case TargetOpcode::G_VAARG:
1569 return legalizeVaArg(MI, MRI, MIRBuilder);
1570 case TargetOpcode::G_LOAD:
1571 case TargetOpcode::G_STORE:
1572 return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
1573 case TargetOpcode::G_SHL:
1574 case TargetOpcode::G_ASHR:
1575 case TargetOpcode::G_LSHR:
1576 return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
1577 case TargetOpcode::G_GLOBAL_VALUE:
1578 return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
1579 case TargetOpcode::G_SBFX:
1580 case TargetOpcode::G_UBFX:
1581 return legalizeBitfieldExtract(MI, MRI, Helper);
1582 case TargetOpcode::G_FSHL:
1583 case TargetOpcode::G_FSHR:
1584 return legalizeFunnelShift(MI, MRI, MIRBuilder, Observer, Helper);
1585 case TargetOpcode::G_ROTR:
1586 return legalizeRotate(MI, MRI, Helper);
1587 case TargetOpcode::G_CTPOP:
1588 return legalizeCTPOP(MI, MRI, Helper);
1589 case TargetOpcode::G_ATOMIC_CMPXCHG:
1590 return legalizeAtomicCmpxchg128(MI, MRI, Helper);
1591 case TargetOpcode::G_CTTZ:
1592 return legalizeCTTZ(MI, Helper);
1593 case TargetOpcode::G_BZERO:
1594 case TargetOpcode::G_MEMCPY:
1595 case TargetOpcode::G_MEMMOVE:
1596 case TargetOpcode::G_MEMSET:
1597 case TargetOpcode::G_MEMSET_INLINE:
1598 return legalizeMemOps(MI, Helper);
1599 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1600 return legalizeExtractVectorElt(MI, MRI, Helper);
1601 case TargetOpcode::G_DYN_STACKALLOC:
1602 return legalizeDynStackAlloc(MI, Helper);
1603 case TargetOpcode::G_PREFETCH:
1604 return legalizePrefetch(MI, Helper);
1605 case TargetOpcode::G_ABS:
1606 return Helper.lowerAbsToCNeg(MI);
1607 case TargetOpcode::G_ICMP:
1608 return legalizeICMP(MI, MRI, MIRBuilder);
1609 case TargetOpcode::G_BITCAST:
1610 return legalizeBitcast(MI, Helper);
1611 case TargetOpcode::G_CONCAT_VECTORS:
1612 return legalizeConcatVectors(MI, MRI, MIRBuilder);
1613 case TargetOpcode::G_FPTRUNC:
1614 // In order to lower f16 to f64 properly, we need to use f32 as an
1615 // intermediary
1616 return legalizeFptrunc(MI, MIRBuilder, MRI);
1617 case TargetOpcode::G_GET_ROUNDING:
1618 return legalizeGetRounding(MI, MIRBuilder, MRI, Helper);
1619 case TargetOpcode::G_SET_ROUNDING:
1620 return legalizeSetRounding(MI, MIRBuilder, MRI, Helper);
1621 }
1622
1623 llvm_unreachable("expected switch to return");
1624}
1625
1626bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
1627 LegalizerHelper &Helper) const {
1628 assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
1629 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1630 // We're trying to handle casts from i1 vectors to scalars but reloading from
1631 // stack.
1632 if (!DstTy.isScalar() || !SrcTy.isVector() ||
1633 SrcTy.getElementType() != LLT::scalar(SizeInBits: 1))
1634 return false;
1635
1636 Helper.createStackStoreLoad(Res: DstReg, Val: SrcReg);
1637 MI.eraseFromParent();
1638 return true;
1639}
1640
1641bool AArch64LegalizerInfo::legalizeFunnelShift(MachineInstr &MI,
1642 MachineRegisterInfo &MRI,
1643 MachineIRBuilder &MIRBuilder,
1644 GISelChangeObserver &Observer,
1645 LegalizerHelper &Helper) const {
1646 assert(MI.getOpcode() == TargetOpcode::G_FSHL ||
1647 MI.getOpcode() == TargetOpcode::G_FSHR);
1648
1649 // Keep as G_FSHR if shift amount is a G_CONSTANT, else use generic
1650 // lowering
1651 Register ShiftNo = MI.getOperand(i: 3).getReg();
1652 LLT ShiftTy = MRI.getType(Reg: ShiftNo);
1653 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ShiftNo, MRI);
1654
1655 // Adjust shift amount according to Opcode (FSHL/FSHR)
1656 // Convert FSHL to FSHR
1657 LLT OperationTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1658 APInt BitWidth(ShiftTy.getSizeInBits(), OperationTy.getSizeInBits(), false);
1659
1660 // Lower non-constant shifts and leave zero shifts to the optimizer.
1661 if (!VRegAndVal || VRegAndVal->Value.urem(RHS: BitWidth) == 0)
1662 return (Helper.lowerFunnelShiftAsShifts(MI) ==
1663 LegalizerHelper::LegalizeResult::Legalized);
1664
1665 APInt Amount = VRegAndVal->Value.urem(RHS: BitWidth);
1666
1667 Amount = MI.getOpcode() == TargetOpcode::G_FSHL ? BitWidth - Amount : Amount;
1668
1669 // If the instruction is G_FSHR, has a 64-bit G_CONSTANT for shift amount
1670 // in the range of 0 <-> BitWidth, it is legal
1671 if (ShiftTy.getSizeInBits() == 64 && MI.getOpcode() == TargetOpcode::G_FSHR &&
1672 VRegAndVal->Value.ult(RHS: BitWidth))
1673 return true;
1674
1675 // Cast the ShiftNumber to a 64-bit type
1676 auto Cast64 = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Amount.zext(width: 64));
1677
1678 if (MI.getOpcode() == TargetOpcode::G_FSHR) {
1679 Observer.changingInstr(MI);
1680 MI.getOperand(i: 3).setReg(Cast64.getReg(Idx: 0));
1681 Observer.changedInstr(MI);
1682 }
1683 // If Opcode is FSHL, remove the FSHL instruction and create a FSHR
1684 // instruction
1685 else if (MI.getOpcode() == TargetOpcode::G_FSHL) {
1686 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FSHR, DstOps: {MI.getOperand(i: 0).getReg()},
1687 SrcOps: {MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getReg(),
1688 Cast64.getReg(Idx: 0)});
1689 MI.eraseFromParent();
1690 }
1691 return true;
1692}
1693
1694bool AArch64LegalizerInfo::legalizeICMP(MachineInstr &MI,
1695 MachineRegisterInfo &MRI,
1696 MachineIRBuilder &MIRBuilder) const {
1697 Register DstReg = MI.getOperand(i: 0).getReg();
1698 Register SrcReg1 = MI.getOperand(i: 2).getReg();
1699 Register SrcReg2 = MI.getOperand(i: 3).getReg();
1700 LLT DstTy = MRI.getType(Reg: DstReg);
1701 LLT SrcTy = MRI.getType(Reg: SrcReg1);
1702
1703 // Check the vector types are legal
1704 if (DstTy.getScalarSizeInBits() != SrcTy.getScalarSizeInBits() ||
1705 DstTy.getNumElements() != SrcTy.getNumElements() ||
1706 (DstTy.getSizeInBits() != 64 && DstTy.getSizeInBits() != 128))
1707 return false;
1708
1709 // Lowers G_ICMP NE => G_ICMP EQ to allow better pattern matching for
1710 // following passes
1711 CmpInst::Predicate Pred = (CmpInst::Predicate)MI.getOperand(i: 1).getPredicate();
1712 if (Pred != CmpInst::ICMP_NE)
1713 return true;
1714 Register CmpReg =
1715 MIRBuilder
1716 .buildICmp(Pred: CmpInst::ICMP_EQ, Res: MRI.getType(Reg: DstReg), Op0: SrcReg1, Op1: SrcReg2)
1717 .getReg(Idx: 0);
1718 MIRBuilder.buildNot(Dst: DstReg, Src0: CmpReg);
1719
1720 MI.eraseFromParent();
1721 return true;
1722}
1723
1724bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
1725 MachineRegisterInfo &MRI,
1726 LegalizerHelper &Helper) const {
1727 // To allow for imported patterns to match, we ensure that the rotate amount
1728 // is 64b with an extension.
1729 Register AmtReg = MI.getOperand(i: 2).getReg();
1730 LLT AmtTy = MRI.getType(Reg: AmtReg);
1731 (void)AmtTy;
1732 assert(AmtTy.isScalar() && "Expected a scalar rotate");
1733 assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
1734 auto NewAmt = Helper.MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 64), Op: AmtReg);
1735 Helper.Observer.changingInstr(MI);
1736 MI.getOperand(i: 2).setReg(NewAmt.getReg(Idx: 0));
1737 Helper.Observer.changedInstr(MI);
1738 return true;
1739}
1740
1741bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1742 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
1743 GISelChangeObserver &Observer) const {
1744 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1745 // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
1746 // G_ADD_LOW instructions.
1747 // By splitting this here, we can optimize accesses in the small code model by
1748 // folding in the G_ADD_LOW into the load/store offset.
1749 auto &GlobalOp = MI.getOperand(i: 1);
1750 // Don't modify an intrinsic call.
1751 if (GlobalOp.isSymbol())
1752 return true;
1753 const auto* GV = GlobalOp.getGlobal();
1754 if (GV->isThreadLocal())
1755 return true; // Don't want to modify TLS vars.
1756
1757 auto &TM = ST->getTargetLowering()->getTargetMachine();
1758 unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
1759
1760 if (OpFlags & AArch64II::MO_GOT)
1761 return true;
1762
1763 auto Offset = GlobalOp.getOffset();
1764 Register DstReg = MI.getOperand(i: 0).getReg();
1765 auto ADRP = MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {})
1766 .addGlobalAddress(GV, Offset, TargetFlags: OpFlags | AArch64II::MO_PAGE);
1767 // Set the regclass on the dest reg too.
1768 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1769
1770 // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
1771 // by creating a MOVK that sets bits 48-63 of the register to (global address
1772 // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
1773 // prevent an incorrect tag being generated during relocation when the
1774 // global appears before the code section. Without the offset, a global at
1775 // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
1776 // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
1777 // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
1778 // instead of `0xf`.
1779 // This assumes that we're in the small code model so we can assume a binary
1780 // size of <= 4GB, which makes the untagged PC relative offset positive. The
1781 // binary must also be loaded into address range [0, 2^48). Both of these
1782 // properties need to be ensured at runtime when using tagged addresses.
1783 if (OpFlags & AArch64II::MO_TAGGED) {
1784 assert(!Offset &&
1785 "Should not have folded in an offset for a tagged global!");
1786 ADRP = MIRBuilder.buildInstr(Opc: AArch64::MOVKXi, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {ADRP})
1787 .addGlobalAddress(GV, Offset: 0x100000000,
1788 TargetFlags: AArch64II::MO_PREL | AArch64II::MO_G3)
1789 .addImm(Val: 48);
1790 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1791 }
1792
1793 MIRBuilder.buildInstr(Opc: AArch64::G_ADD_LOW, DstOps: {DstReg}, SrcOps: {ADRP})
1794 .addGlobalAddress(GV, Offset,
1795 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1796 MI.eraseFromParent();
1797 return true;
1798}
1799
1800bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1801 MachineInstr &MI) const {
1802 MachineIRBuilder &MIB = Helper.MIRBuilder;
1803 MachineRegisterInfo &MRI = *MIB.getMRI();
1804
1805 auto LowerUnaryOp = [&MI, &MIB](unsigned Opcode) {
1806 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1807 MI.eraseFromParent();
1808 return true;
1809 };
1810 auto LowerBinOp = [&MI, &MIB](unsigned Opcode) {
1811 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1812 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
1813 MI.eraseFromParent();
1814 return true;
1815 };
1816 auto LowerTriOp = [&MI, &MIB](unsigned Opcode) {
1817 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1818 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4)});
1819 MI.eraseFromParent();
1820 return true;
1821 };
1822
1823 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1824 switch (IntrinsicID) {
1825 case Intrinsic::vacopy: {
1826 unsigned PtrSize = ST->isTargetILP32() ? 4 : 8;
1827 unsigned VaListSize =
1828 (ST->isTargetDarwin() || ST->isTargetWindows())
1829 ? PtrSize
1830 : ST->isTargetILP32() ? 20 : 32;
1831
1832 MachineFunction &MF = *MI.getMF();
1833 auto Val = MF.getRegInfo().createGenericVirtualRegister(
1834 Ty: LLT::integer(SizeInBits: VaListSize * 8));
1835 MIB.buildLoad(Res: Val, Addr: MI.getOperand(i: 2),
1836 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1837 F: MachineMemOperand::MOLoad,
1838 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1839 MIB.buildStore(Val, Addr: MI.getOperand(i: 1),
1840 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1841 F: MachineMemOperand::MOStore,
1842 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1843 MI.eraseFromParent();
1844 return true;
1845 }
1846 case Intrinsic::get_dynamic_area_offset: {
1847 MIB.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: 0);
1848 MI.eraseFromParent();
1849 return true;
1850 }
1851 case Intrinsic::aarch64_mops_memset_tag: {
1852 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1853 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
1854 // the instruction).
1855 auto &Value = MI.getOperand(i: 3);
1856 Register ExtValueReg = MIB.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
1857 Value.setReg(ExtValueReg);
1858 return true;
1859 }
1860 case Intrinsic::aarch64_prefetch: {
1861 auto &AddrVal = MI.getOperand(i: 1);
1862
1863 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1864 int64_t Target = MI.getOperand(i: 3).getImm();
1865 int64_t IsStream = MI.getOperand(i: 4).getImm();
1866 int64_t IsData = MI.getOperand(i: 5).getImm();
1867
1868 unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
1869 (!IsData << 3) | // IsDataCache bit
1870 (Target << 1) | // Cache level bits
1871 (unsigned)IsStream; // Stream bit
1872
1873 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
1874 MI.eraseFromParent();
1875 return true;
1876 }
1877 case Intrinsic::aarch64_range_prefetch: {
1878 auto &AddrVal = MI.getOperand(i: 1);
1879
1880 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1881 int64_t IsStream = MI.getOperand(i: 3).getImm();
1882 unsigned PrfOp = (IsStream << 2) | IsWrite;
1883
1884 MIB.buildInstr(Opcode: AArch64::G_AARCH64_RANGE_PREFETCH)
1885 .addImm(Val: PrfOp)
1886 .add(MO: AddrVal)
1887 .addUse(RegNo: MI.getOperand(i: 4).getReg()); // Metadata
1888 MI.eraseFromParent();
1889 return true;
1890 }
1891 case Intrinsic::aarch64_prefetch_ir: {
1892 auto &AddrVal = MI.getOperand(i: 1);
1893 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: 24).add(MO: AddrVal);
1894 MI.eraseFromParent();
1895 return true;
1896 }
1897 case Intrinsic::aarch64_neon_uaddv:
1898 case Intrinsic::aarch64_neon_saddv:
1899 case Intrinsic::aarch64_neon_umaxv:
1900 case Intrinsic::aarch64_neon_smaxv:
1901 case Intrinsic::aarch64_neon_uminv:
1902 case Intrinsic::aarch64_neon_sminv: {
1903 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1904 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1905 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1906
1907 auto OldDst = MI.getOperand(i: 0).getReg();
1908 auto OldDstTy = MRI.getType(Reg: OldDst);
1909 LLT NewDstTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType();
1910 if (OldDstTy == NewDstTy)
1911 return true;
1912
1913 auto NewDst = MRI.createGenericVirtualRegister(Ty: NewDstTy);
1914
1915 Helper.Observer.changingInstr(MI);
1916 MI.getOperand(i: 0).setReg(NewDst);
1917 Helper.Observer.changedInstr(MI);
1918
1919 MIB.setInsertPt(MBB&: MIB.getMBB(), II: ++MIB.getInsertPt());
1920 MIB.buildExtOrTrunc(ExtOpc: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1921 Res: OldDst, Op: NewDst);
1922
1923 return true;
1924 }
1925 case Intrinsic::aarch64_neon_uaddlp:
1926 case Intrinsic::aarch64_neon_saddlp: {
1927 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1928 ? AArch64::G_UADDLP
1929 : AArch64::G_SADDLP;
1930 MIB.buildInstr(Opc, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1931 MI.eraseFromParent();
1932
1933 return true;
1934 }
1935 case Intrinsic::aarch64_neon_uaddlv:
1936 case Intrinsic::aarch64_neon_saddlv: {
1937 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1938 ? AArch64::G_UADDLV
1939 : AArch64::G_SADDLV;
1940 Register DstReg = MI.getOperand(i: 0).getReg();
1941 Register SrcReg = MI.getOperand(i: 2).getReg();
1942 LLT DstTy = MRI.getType(Reg: DstReg);
1943
1944 LLT MidTy, ExtTy;
1945 if (DstTy.isScalar() && DstTy.getScalarSizeInBits() <= 32) {
1946 ExtTy = LLT::integer(SizeInBits: 32);
1947 MidTy = LLT::fixed_vector(NumElements: 4, ScalarTy: ExtTy);
1948 } else {
1949 ExtTy = LLT::integer(SizeInBits: 64);
1950 MidTy = LLT::fixed_vector(NumElements: 2, ScalarTy: ExtTy);
1951 }
1952
1953 Register MidReg =
1954 MIB.buildInstr(Opc, DstOps: {MidTy}, SrcOps: {SrcReg})->getOperand(i: 0).getReg();
1955 Register ZeroReg =
1956 MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0)->getOperand(i: 0).getReg();
1957 Register ExtReg = MIB.buildInstr(Opc: AArch64::G_EXTRACT_VECTOR_ELT, DstOps: {ExtTy},
1958 SrcOps: {MidReg, ZeroReg})
1959 .getReg(Idx: 0);
1960
1961 if (DstTy.getScalarSizeInBits() < 32)
1962 MIB.buildTrunc(Res: DstReg, Op: ExtReg);
1963 else
1964 MIB.buildCopy(Res: DstReg, Op: ExtReg);
1965
1966 MI.eraseFromParent();
1967
1968 return true;
1969 }
1970 case Intrinsic::aarch64_neon_smax:
1971 return LowerBinOp(TargetOpcode::G_SMAX);
1972 case Intrinsic::aarch64_neon_smin:
1973 return LowerBinOp(TargetOpcode::G_SMIN);
1974 case Intrinsic::aarch64_neon_umax:
1975 return LowerBinOp(TargetOpcode::G_UMAX);
1976 case Intrinsic::aarch64_neon_umin:
1977 return LowerBinOp(TargetOpcode::G_UMIN);
1978 case Intrinsic::aarch64_neon_fmax:
1979 return LowerBinOp(TargetOpcode::G_FMAXIMUM);
1980 case Intrinsic::aarch64_neon_fmin:
1981 return LowerBinOp(TargetOpcode::G_FMINIMUM);
1982 case Intrinsic::aarch64_neon_fmaxnm:
1983 return LowerBinOp(TargetOpcode::G_FMAXNUM);
1984 case Intrinsic::aarch64_neon_fminnm:
1985 return LowerBinOp(TargetOpcode::G_FMINNUM);
1986 case Intrinsic::aarch64_neon_pmul:
1987 return LowerBinOp(TargetOpcode::G_CLMUL);
1988 case Intrinsic::aarch64_neon_pmull:
1989 case Intrinsic::aarch64_neon_pmull64:
1990 return LowerBinOp(AArch64::G_PMULL);
1991 case Intrinsic::aarch64_neon_smull:
1992 return LowerBinOp(AArch64::G_SMULL);
1993 case Intrinsic::aarch64_neon_umull:
1994 return LowerBinOp(AArch64::G_UMULL);
1995 case Intrinsic::aarch64_neon_sabd:
1996 return LowerBinOp(TargetOpcode::G_ABDS);
1997 case Intrinsic::aarch64_neon_uabd:
1998 return LowerBinOp(TargetOpcode::G_ABDU);
1999 case Intrinsic::aarch64_neon_uhadd:
2000 return LowerBinOp(TargetOpcode::G_UAVGFLOOR);
2001 case Intrinsic::aarch64_neon_urhadd:
2002 return LowerBinOp(TargetOpcode::G_UAVGCEIL);
2003 case Intrinsic::aarch64_neon_shadd:
2004 return LowerBinOp(TargetOpcode::G_SAVGFLOOR);
2005 case Intrinsic::aarch64_neon_srhadd:
2006 return LowerBinOp(TargetOpcode::G_SAVGCEIL);
2007 case Intrinsic::aarch64_neon_sqshrn: {
2008 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2009 return true;
2010 // Create right shift instruction. Store the output register in Shr.
2011 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2012 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2013 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2014 // Build the narrow intrinsic, taking in Shr.
2015 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2016 MI.eraseFromParent();
2017 return true;
2018 }
2019 case Intrinsic::aarch64_neon_sqshrun: {
2020 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2021 return true;
2022 // Create right shift instruction. Store the output register in Shr.
2023 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2024 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2025 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2026 // Build the narrow intrinsic, taking in Shr.
2027 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2028 MI.eraseFromParent();
2029 return true;
2030 }
2031 case Intrinsic::aarch64_neon_sqrshrn: {
2032 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2033 return true;
2034 // Create right shift instruction. Store the output register in Shr.
2035 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2036 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2037 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2038 // Build the narrow intrinsic, taking in Shr.
2039 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2040 MI.eraseFromParent();
2041 return true;
2042 }
2043 case Intrinsic::aarch64_neon_sqrshrun: {
2044 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2045 return true;
2046 // Create right shift instruction. Store the output register in Shr.
2047 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2048 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2049 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2050 // Build the narrow intrinsic, taking in Shr.
2051 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2052 MI.eraseFromParent();
2053 return true;
2054 }
2055 case Intrinsic::aarch64_neon_uqrshrn: {
2056 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2057 return true;
2058 // Create right shift instruction. Store the output register in Shr.
2059 auto Shr = MIB.buildInstr(Opc: AArch64::G_URSHR_I,
2060 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2061 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2062 // Build the narrow intrinsic, taking in Shr.
2063 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2064 MI.eraseFromParent();
2065 return true;
2066 }
2067 case Intrinsic::aarch64_neon_uqshrn: {
2068 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2069 return true;
2070 // Create right shift instruction. Store the output register in Shr.
2071 auto Shr = MIB.buildInstr(Opc: AArch64::G_VLSHR,
2072 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2073 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2074 // Build the narrow intrinsic, taking in Shr.
2075 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2076 MI.eraseFromParent();
2077 return true;
2078 }
2079 case Intrinsic::aarch64_neon_sqshlu: {
2080 // Check if last operand is constant vector dup
2081 auto ShiftAmount =
2082 isConstantOrConstantSplatVector(Def: MI.getOperand(i: 3).getReg(), MRI);
2083 if (ShiftAmount) {
2084 // If so, create a new intrinsic with the correct shift amount
2085 MIB.buildInstr(Opc: AArch64::G_SQSHLU_I, DstOps: {MI.getOperand(i: 0)},
2086 SrcOps: {MI.getOperand(i: 2)})
2087 .addImm(Val: ShiftAmount->getSExtValue());
2088 MI.eraseFromParent();
2089 return true;
2090 }
2091 return false;
2092 }
2093 case Intrinsic::aarch64_neon_vsli: {
2094 MIB.buildInstr(
2095 Opc: AArch64::G_SLI, DstOps: {MI.getOperand(i: 0)},
2096 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2097 MI.eraseFromParent();
2098 break;
2099 }
2100 case Intrinsic::aarch64_neon_vsri: {
2101 MIB.buildInstr(
2102 Opc: AArch64::G_SRI, DstOps: {MI.getOperand(i: 0)},
2103 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2104 MI.eraseFromParent();
2105 break;
2106 }
2107 case Intrinsic::aarch64_neon_abs: {
2108 // Lower the intrinsic to G_ABS.
2109 MIB.buildInstr(Opc: TargetOpcode::G_ABS, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
2110 MI.eraseFromParent();
2111 return true;
2112 }
2113 case Intrinsic::aarch64_neon_addhn:
2114 return LowerBinOp(AArch64::G_ADDHN);
2115 case Intrinsic::aarch64_neon_sqadd: {
2116 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2117 return LowerBinOp(TargetOpcode::G_SADDSAT);
2118 break;
2119 }
2120 case Intrinsic::aarch64_neon_sqsub: {
2121 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2122 return LowerBinOp(TargetOpcode::G_SSUBSAT);
2123 break;
2124 }
2125 case Intrinsic::aarch64_neon_uqadd: {
2126 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2127 return LowerBinOp(TargetOpcode::G_UADDSAT);
2128 break;
2129 }
2130 case Intrinsic::aarch64_neon_uqsub: {
2131 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2132 return LowerBinOp(TargetOpcode::G_USUBSAT);
2133 break;
2134 }
2135 case Intrinsic::aarch64_neon_udot:
2136 return LowerTriOp(AArch64::G_UDOT);
2137 case Intrinsic::aarch64_neon_sdot:
2138 return LowerTriOp(AArch64::G_SDOT);
2139 case Intrinsic::aarch64_neon_usdot:
2140 return LowerTriOp(AArch64::G_USDOT);
2141 case Intrinsic::aarch64_neon_sqxtn:
2142 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_S);
2143 case Intrinsic::aarch64_neon_sqxtun:
2144 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_U);
2145 case Intrinsic::aarch64_neon_uqxtn:
2146 return LowerUnaryOp(TargetOpcode::G_TRUNC_USAT_U);
2147 case Intrinsic::aarch64_neon_fcvtzu:
2148 return LowerUnaryOp(TargetOpcode::G_FPTOUI_SAT);
2149 case Intrinsic::aarch64_neon_fcvtzs:
2150 return LowerUnaryOp(TargetOpcode::G_FPTOSI_SAT);
2151 case Intrinsic::aarch64_neon_cls:
2152 return LowerUnaryOp(TargetOpcode::G_CTLS);
2153
2154 case Intrinsic::vector_reverse:
2155 // TODO: Add support for vector_reverse
2156 return false;
2157 }
2158
2159 return true;
2160}
2161
2162bool AArch64LegalizerInfo::legalizeShlAshrLshr(
2163 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2164 GISelChangeObserver &Observer) const {
2165 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
2166 MI.getOpcode() == TargetOpcode::G_LSHR ||
2167 MI.getOpcode() == TargetOpcode::G_SHL);
2168 // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
2169 // imported patterns can select it later. Either way, it will be legal.
2170 Register AmtReg = MI.getOperand(i: 2).getReg();
2171 LLT AmtRegEltTy = MRI.getType(Reg: AmtReg).getScalarType();
2172 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI);
2173 if (!VRegAndVal)
2174 return true;
2175 // Check the shift amount is in range for an immediate form.
2176 int64_t Amount = VRegAndVal->Value.getSExtValue();
2177 if (Amount > 31)
2178 return true; // This will have to remain a register variant.
2179 auto ExtCst =
2180 MIRBuilder.buildConstant(Res: AmtRegEltTy.changeElementSize(NewEltSize: 64), Val: Amount);
2181 Observer.changingInstr(MI);
2182 MI.getOperand(i: 2).setReg(ExtCst.getReg(Idx: 0));
2183 Observer.changedInstr(MI);
2184 return true;
2185}
2186
2187static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset,
2188 MachineRegisterInfo &MRI) {
2189 Base = Root;
2190 Offset = 0;
2191
2192 Register NewBase;
2193 int64_t NewOffset;
2194 if (mi_match(R: Root, MRI, P: m_GPtrAdd(L: m_Reg(R&: NewBase), R: m_ICst(Cst&: NewOffset))) &&
2195 isShiftedInt<7, 3>(x: NewOffset)) {
2196 Base = NewBase;
2197 Offset = NewOffset;
2198 }
2199}
2200
2201// FIXME: This should be removed and replaced with the generic bitcast legalize
2202// action.
2203bool AArch64LegalizerInfo::legalizeLoadStore(
2204 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2205 GISelChangeObserver &Observer) const {
2206 assert(MI.getOpcode() == TargetOpcode::G_STORE ||
2207 MI.getOpcode() == TargetOpcode::G_LOAD);
2208 // Here we just try to handle vector loads/stores where our value type might
2209 // have pointer elements, which the SelectionDAG importer can't handle. To
2210 // allow the existing patterns for s64 to fire for p0, we just try to bitcast
2211 // the value to use s64 types.
2212
2213 // Custom legalization requires the instruction, if not deleted, must be fully
2214 // legalized. In order to allow further legalization of the inst, we create
2215 // a new instruction and erase the existing one.
2216
2217 Register ValReg = MI.getOperand(i: 0).getReg();
2218 const LLT ValTy = MRI.getType(Reg: ValReg);
2219
2220 if (ValTy == LLT::scalar(SizeInBits: 128)) {
2221
2222 AtomicOrdering Ordering = (*MI.memoperands_begin())->getSuccessOrdering();
2223 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2224 bool IsLoadAcquire = IsLoad && Ordering == AtomicOrdering::Acquire;
2225 bool IsStoreRelease = !IsLoad && Ordering == AtomicOrdering::Release;
2226 bool IsRcpC3 =
2227 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
2228
2229 LLT s64 = LLT::integer(SizeInBits: 64);
2230
2231 unsigned Opcode;
2232 if (IsRcpC3) {
2233 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
2234 } else {
2235 // For LSE2, loads/stores should have been converted to monotonic and had
2236 // a fence inserted after them.
2237 assert(Ordering == AtomicOrdering::Monotonic ||
2238 Ordering == AtomicOrdering::Unordered);
2239 assert(ST->hasLSE2() && "ldp/stp not single copy atomic without +lse2");
2240
2241 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
2242 }
2243
2244 MachineInstrBuilder NewI;
2245 if (IsLoad) {
2246 NewI = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {s64, s64}, SrcOps: {});
2247 MIRBuilder.buildMergeLikeInstr(
2248 Res: ValReg, Ops: {NewI->getOperand(i: 0), NewI->getOperand(i: 1)});
2249 } else {
2250 auto Split = MIRBuilder.buildUnmerge(Res: s64, Op: MI.getOperand(i: 0));
2251 NewI = MIRBuilder.buildInstr(
2252 Opc: Opcode, DstOps: {}, SrcOps: {Split->getOperand(i: 0), Split->getOperand(i: 1)});
2253 }
2254
2255 if (IsRcpC3) {
2256 NewI.addUse(RegNo: MI.getOperand(i: 1).getReg());
2257 } else {
2258 Register Base;
2259 int Offset;
2260 matchLDPSTPAddrMode(Root: MI.getOperand(i: 1).getReg(), Base, Offset, MRI);
2261 NewI.addUse(RegNo: Base);
2262 NewI.addImm(Val: Offset / 8);
2263 }
2264
2265 NewI.cloneMemRefs(OtherMI: MI);
2266 constrainSelectedInstRegOperands(I&: *NewI, TII: *ST->getInstrInfo(),
2267 TRI: *MRI.getTargetRegisterInfo(),
2268 RBI: *ST->getRegBankInfo());
2269 MI.eraseFromParent();
2270 return true;
2271 }
2272
2273 if (!ValTy.isPointerVector() ||
2274 ValTy.getElementType().getAddressSpace() != 0) {
2275 LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
2276 return false;
2277 }
2278
2279 unsigned PtrSize = ValTy.getElementType().getSizeInBits();
2280 const LLT NewTy = LLT::vector(EC: ValTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: PtrSize));
2281 auto &MMO = **MI.memoperands_begin();
2282 MMO.setType(NewTy);
2283
2284 if (MI.getOpcode() == TargetOpcode::G_STORE) {
2285 auto Bitcast = MIRBuilder.buildBitcast(Dst: NewTy, Src: ValReg);
2286 MIRBuilder.buildStore(Val: Bitcast.getReg(Idx: 0), Addr: MI.getOperand(i: 1), MMO);
2287 } else {
2288 auto NewLoad = MIRBuilder.buildLoad(Res: NewTy, Addr: MI.getOperand(i: 1), MMO);
2289 MIRBuilder.buildBitcast(Dst: ValReg, Src: NewLoad);
2290 }
2291 MI.eraseFromParent();
2292 return true;
2293}
2294
2295bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
2296 MachineRegisterInfo &MRI,
2297 MachineIRBuilder &MIRBuilder) const {
2298 MachineFunction &MF = MIRBuilder.getMF();
2299 Align Alignment(MI.getOperand(i: 2).getImm());
2300 Register Dst = MI.getOperand(i: 0).getReg();
2301 Register ListPtr = MI.getOperand(i: 1).getReg();
2302
2303 LLT PtrTy = MRI.getType(Reg: ListPtr);
2304 LLT IntPtrTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
2305
2306 const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
2307 const Align PtrAlign = Align(PtrSize);
2308 auto List = MIRBuilder.buildLoad(
2309 Res: PtrTy, Addr: ListPtr,
2310 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2311 MemTy: PtrTy, BaseAlignment: PtrAlign));
2312
2313 MachineInstrBuilder DstPtr;
2314 if (Alignment > PtrAlign) {
2315 // Realign the list to the actual required alignment.
2316 auto AlignMinus1 =
2317 MIRBuilder.buildConstant(Res: IntPtrTy, Val: Alignment.value() - 1);
2318 auto ListTmp = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: List, Op1: AlignMinus1.getReg(Idx: 0));
2319 DstPtr = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: ListTmp, NumBits: Log2(A: Alignment));
2320 } else
2321 DstPtr = List;
2322
2323 LLT ValTy = MRI.getType(Reg: Dst);
2324 uint64_t ValSize = ValTy.getSizeInBits() / 8;
2325 MIRBuilder.buildLoad(
2326 Res: Dst, Addr: DstPtr,
2327 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), F: MachineMemOperand::MOLoad,
2328 MemTy: ValTy, BaseAlignment: std::max(a: Alignment, b: PtrAlign)));
2329
2330 auto Size = MIRBuilder.buildConstant(Res: IntPtrTy, Val: alignTo(Size: ValSize, A: PtrAlign));
2331
2332 auto NewList = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: DstPtr, Op1: Size.getReg(Idx: 0));
2333
2334 MIRBuilder.buildStore(Val: NewList, Addr: ListPtr,
2335 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
2336 F: MachineMemOperand::MOStore,
2337 MemTy: PtrTy, BaseAlignment: PtrAlign));
2338
2339 MI.eraseFromParent();
2340 return true;
2341}
2342
2343bool AArch64LegalizerInfo::legalizeBitfieldExtract(
2344 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2345 // Only legal if we can select immediate forms.
2346 // TODO: Lower this otherwise.
2347 return getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI) &&
2348 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
2349}
2350
2351bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
2352 MachineRegisterInfo &MRI,
2353 LegalizerHelper &Helper) const {
2354 // When there is no integer popcount instruction (FEAT_CSSC isn't available),
2355 // it can be more efficiently lowered to the following sequence that uses
2356 // AdvSIMD registers/instructions as long as the copies to/from the AdvSIMD
2357 // registers are cheap.
2358 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
2359 // CNT V0.8B, V0.8B // 8xbyte pop-counts
2360 // ADDV B0, V0.8B // sum 8xbyte pop-counts
2361 // UMOV X0, V0.B[0] // copy byte result back to integer reg
2362 //
2363 // For 128 bit vector popcounts, we lower to the following sequence:
2364 // cnt.16b v0, v0 // v8s16, v4s32, v2s64
2365 // uaddlp.8h v0, v0 // v8s16, v4s32, v2s64
2366 // uaddlp.4s v0, v0 // v4s32, v2s64
2367 // uaddlp.2d v0, v0 // v2s64
2368 //
2369 // For 64 bit vector popcounts, we lower to the following sequence:
2370 // cnt.8b v0, v0 // v4s16, v2s32
2371 // uaddlp.4h v0, v0 // v4s16, v2s32
2372 // uaddlp.2s v0, v0 // v2s32
2373
2374 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2375 Register Dst = MI.getOperand(i: 0).getReg();
2376 Register Val = MI.getOperand(i: 1).getReg();
2377 LLT Ty = MRI.getType(Reg: Val);
2378
2379 LLT i64 = LLT::integer(SizeInBits: 64);
2380 LLT i32 = LLT::integer(SizeInBits: 32);
2381 LLT i16 = LLT::integer(SizeInBits: 16);
2382 LLT i8 = LLT::integer(SizeInBits: 8);
2383 unsigned Size = Ty.getSizeInBits();
2384
2385 assert(Ty == MRI.getType(Dst) &&
2386 "Expected src and dst to have the same type!");
2387
2388 if (ST->hasCSSC() && Ty.isScalar() && Size == 128) {
2389
2390 auto Split = MIRBuilder.buildUnmerge(Res: i64, Op: Val);
2391 auto CTPOP1 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 0));
2392 auto CTPOP2 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 1));
2393 auto Add = MIRBuilder.buildAdd(Dst: i64, Src0: CTPOP1, Src1: CTPOP2);
2394
2395 MIRBuilder.buildZExt(Res: Dst, Op: Add);
2396 MI.eraseFromParent();
2397 return true;
2398 }
2399
2400 if (!ST->hasNEON() ||
2401 MI.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
2402 // Use generic lowering when custom lowering is not possible.
2403 return Ty.isScalar() && (Size == 32 || Size == 64) &&
2404 Helper.lowerBitCount(MI) ==
2405 LegalizerHelper::LegalizeResult::Legalized;
2406 }
2407
2408 // Pre-conditioning: widen Val up to the nearest vector type.
2409 // s32,s64,v4s16,v2s32 -> v8i8
2410 // v8s16,v4s32,v2s64 -> v16i8
2411 LLT VTy = Size == 128 ? LLT::fixed_vector(NumElements: 16, ScalarTy: i8) : LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
2412 if (Ty.isScalar()) {
2413 assert((Size == 32 || Size == 64 || Size == 128) && "Expected only 32, 64, or 128 bit scalars!");
2414 if (Size == 32) {
2415 Val = MIRBuilder.buildZExt(Res: i64, Op: Val).getReg(Idx: 0);
2416 }
2417 }
2418 Val = MIRBuilder.buildBitcast(Dst: VTy, Src: Val).getReg(Idx: 0);
2419
2420 // Count bits in each byte-sized lane.
2421 auto CTPOP = MIRBuilder.buildCTPOP(Dst: VTy, Src0: Val);
2422
2423 // Sum across lanes.
2424 if (ST->hasDotProd() && Ty.isVector() && Ty.getNumElements() >= 2 &&
2425 Ty.getScalarSizeInBits() != 16) {
2426 LLT Dt = Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64) ? LLT::fixed_vector(NumElements: 4, ScalarTy: i32) : Ty;
2427 auto Zeros = MIRBuilder.buildConstant(Res: Dt, Val: 0);
2428 auto Ones = MIRBuilder.buildConstant(Res: VTy, Val: 1);
2429 MachineInstrBuilder Sum;
2430
2431 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2432 auto UDOT =
2433 MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2434 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UADDLP, DstOps: {Ty}, SrcOps: {UDOT});
2435 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2436 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2437 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2438 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2439 } else {
2440 llvm_unreachable("unexpected vector shape");
2441 }
2442
2443 Sum->getOperand(i: 0).setReg(Dst);
2444 MI.eraseFromParent();
2445 return true;
2446 }
2447
2448 Register HSum = CTPOP.getReg(Idx: 0);
2449 unsigned Opc;
2450 SmallVector<LLT> HAddTys;
2451 if (Ty.isScalar()) {
2452 Opc = Intrinsic::aarch64_neon_uaddlv;
2453 HAddTys.push_back(Elt: i32);
2454 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: i16)) {
2455 Opc = Intrinsic::aarch64_neon_uaddlp;
2456 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2457 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2458 Opc = Intrinsic::aarch64_neon_uaddlp;
2459 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2460 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2461 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2462 Opc = Intrinsic::aarch64_neon_uaddlp;
2463 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2464 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2465 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i64));
2466 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i16)) {
2467 Opc = Intrinsic::aarch64_neon_uaddlp;
2468 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2469 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2470 Opc = Intrinsic::aarch64_neon_uaddlp;
2471 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2472 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i32));
2473 } else
2474 llvm_unreachable("unexpected vector shape");
2475 MachineInstrBuilder UADD;
2476 for (LLT HTy : HAddTys) {
2477 UADD = MIRBuilder.buildIntrinsic(ID: Opc, Res: {HTy}).addUse(RegNo: HSum);
2478 HSum = UADD.getReg(Idx: 0);
2479 }
2480
2481 // Post-conditioning.
2482 if (Ty.isScalar() && (Size == 64 || Size == 128))
2483 MIRBuilder.buildZExt(Res: Dst, Op: UADD);
2484 else
2485 UADD->getOperand(i: 0).setReg(Dst);
2486 MI.eraseFromParent();
2487 return true;
2488}
2489
2490bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
2491 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2492 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2493 LLT i64 = LLT::integer(SizeInBits: 64);
2494 auto Addr = MI.getOperand(i: 1).getReg();
2495 auto DesiredI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 2));
2496 auto NewI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 3));
2497 auto DstLo = MRI.createGenericVirtualRegister(Ty: i64);
2498 auto DstHi = MRI.createGenericVirtualRegister(Ty: i64);
2499
2500 MachineInstrBuilder CAS;
2501 if (ST->hasLSE()) {
2502 // We have 128-bit CASP instructions taking XSeqPair registers, which are
2503 // s128. We need the merge/unmerge to bracket the expansion and pair up with
2504 // the rest of the MIR so we must reassemble the extracted registers into a
2505 // 128-bit known-regclass one with code like this:
2506 //
2507 // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
2508 // %out = CASP %in1, ...
2509 // %OldLo = G_EXTRACT %out, 0
2510 // %OldHi = G_EXTRACT %out, 64
2511 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2512 unsigned Opcode;
2513 switch (Ordering) {
2514 case AtomicOrdering::Acquire:
2515 Opcode = AArch64::CASPAX;
2516 break;
2517 case AtomicOrdering::Release:
2518 Opcode = AArch64::CASPLX;
2519 break;
2520 case AtomicOrdering::AcquireRelease:
2521 case AtomicOrdering::SequentiallyConsistent:
2522 Opcode = AArch64::CASPALX;
2523 break;
2524 default:
2525 Opcode = AArch64::CASPX;
2526 break;
2527 }
2528
2529 LLT s128 = LLT::integer(SizeInBits: 128);
2530 auto CASDst = MRI.createGenericVirtualRegister(Ty: s128);
2531 auto CASDesired = MRI.createGenericVirtualRegister(Ty: s128);
2532 auto CASNew = MRI.createGenericVirtualRegister(Ty: s128);
2533 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASDesired}, SrcOps: {})
2534 .addUse(RegNo: DesiredI->getOperand(i: 0).getReg())
2535 .addImm(Val: AArch64::sube64)
2536 .addUse(RegNo: DesiredI->getOperand(i: 1).getReg())
2537 .addImm(Val: AArch64::subo64);
2538 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASNew}, SrcOps: {})
2539 .addUse(RegNo: NewI->getOperand(i: 0).getReg())
2540 .addImm(Val: AArch64::sube64)
2541 .addUse(RegNo: NewI->getOperand(i: 1).getReg())
2542 .addImm(Val: AArch64::subo64);
2543
2544 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {CASDst}, SrcOps: {CASDesired, CASNew, Addr});
2545
2546 MIRBuilder.buildExtract(Res: {DstLo}, Src: {CASDst}, Index: 0);
2547 MIRBuilder.buildExtract(Res: {DstHi}, Src: {CASDst}, Index: 64);
2548 } else {
2549 // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
2550 // can take arbitrary registers so it just has the normal GPR64 operands the
2551 // rest of AArch64 is expecting.
2552 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2553 unsigned Opcode;
2554 switch (Ordering) {
2555 case AtomicOrdering::Acquire:
2556 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
2557 break;
2558 case AtomicOrdering::Release:
2559 Opcode = AArch64::CMP_SWAP_128_RELEASE;
2560 break;
2561 case AtomicOrdering::AcquireRelease:
2562 case AtomicOrdering::SequentiallyConsistent:
2563 Opcode = AArch64::CMP_SWAP_128;
2564 break;
2565 default:
2566 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
2567 break;
2568 }
2569
2570 auto Scratch = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2571 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {DstLo, DstHi, Scratch},
2572 SrcOps: {Addr, DesiredI->getOperand(i: 0),
2573 DesiredI->getOperand(i: 1), NewI->getOperand(i: 0),
2574 NewI->getOperand(i: 1)});
2575 }
2576
2577 CAS.cloneMemRefs(OtherMI: MI);
2578 constrainSelectedInstRegOperands(I&: *CAS, TII: *ST->getInstrInfo(),
2579 TRI: *MRI.getTargetRegisterInfo(),
2580 RBI: *ST->getRegBankInfo());
2581
2582 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {DstLo, DstHi});
2583 MI.eraseFromParent();
2584 return true;
2585}
2586
2587bool AArch64LegalizerInfo::legalizeCTTZ(MachineInstr &MI,
2588 LegalizerHelper &Helper) const {
2589 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2590 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2591 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2592 auto BitReverse = MIRBuilder.buildBitReverse(Dst: Ty, Src: MI.getOperand(i: 1));
2593 MIRBuilder.buildCTLZ(Dst: MI.getOperand(i: 0).getReg(), Src0: BitReverse);
2594 MI.eraseFromParent();
2595 return true;
2596}
2597
2598bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
2599 LegalizerHelper &Helper) const {
2600 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2601
2602 // Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
2603 if (MI.getOpcode() == TargetOpcode::G_MEMSET ||
2604 MI.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2605 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
2606 // the instruction).
2607 auto &Value = MI.getOperand(i: 1);
2608 Register ExtValueReg =
2609 MIRBuilder.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
2610 Value.setReg(ExtValueReg);
2611 return true;
2612 }
2613
2614 return false;
2615}
2616
2617bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2618 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2619 const GExtractVectorElement *Element = cast<GExtractVectorElement>(Val: &MI);
2620 auto VRegAndVal =
2621 getIConstantVRegValWithLookThrough(VReg: Element->getIndexReg(), MRI);
2622 if (VRegAndVal)
2623 return true;
2624 LLT VecTy = MRI.getType(Reg: Element->getVectorReg());
2625 if (VecTy.isScalableVector())
2626 return true;
2627 return Helper.lowerExtractInsertVectorElt(MI) !=
2628 LegalizerHelper::LegalizeResult::UnableToLegalize;
2629}
2630
2631bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2632 MachineInstr &MI, LegalizerHelper &Helper) const {
2633 MachineFunction &MF = *MI.getParent()->getParent();
2634 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2635 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2636
2637 // If stack probing is not enabled for this function, use the default
2638 // lowering.
2639 if (!MF.getFunction().hasFnAttribute(Kind: "probe-stack") ||
2640 MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() !=
2641 "inline-asm") {
2642 Helper.lowerDynStackAlloc(MI);
2643 return true;
2644 }
2645
2646 Register Dst = MI.getOperand(i: 0).getReg();
2647 Register AllocSize = MI.getOperand(i: 1).getReg();
2648 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
2649
2650 assert(MRI.getType(Dst) == LLT::pointer(0, 64) &&
2651 "Unexpected type for dynamic alloca");
2652 assert(MRI.getType(AllocSize) == LLT::scalar(64) &&
2653 "Unexpected type for dynamic alloca");
2654
2655 LLT PtrTy = MRI.getType(Reg: Dst);
2656 Register SPReg =
2657 Helper.getTargetLowering().getStackPointerRegisterToSaveRestore();
2658 Register SPTmp =
2659 Helper.getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
2660 auto NewMI =
2661 MIRBuilder.buildInstr(Opc: AArch64::PROBED_STACKALLOC_DYN, DstOps: {}, SrcOps: {SPTmp});
2662 MRI.setRegClass(Reg: NewMI.getReg(Idx: 0), RC: &AArch64::GPR64commonRegClass);
2663 MIRBuilder.setInsertPt(MBB&: *NewMI->getParent(), II: NewMI);
2664 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
2665
2666 MI.eraseFromParent();
2667 return true;
2668}
2669
2670bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
2671 LegalizerHelper &Helper) const {
2672 MachineIRBuilder &MIB = Helper.MIRBuilder;
2673 auto &AddrVal = MI.getOperand(i: 0);
2674
2675 int64_t IsWrite = MI.getOperand(i: 1).getImm();
2676 int64_t Locality = MI.getOperand(i: 2).getImm();
2677 int64_t IsData = MI.getOperand(i: 3).getImm();
2678
2679 bool IsStream = Locality == 0;
2680 if (Locality != 0) {
2681 assert(Locality <= 3 && "Prefetch locality out-of-range");
2682 // The locality degree is the opposite of the cache speed.
2683 // Put the number the other way around.
2684 // The encoding starts at 0 for level 1
2685 Locality = 3 - Locality;
2686 }
2687
2688 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2689
2690 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
2691 MI.eraseFromParent();
2692 return true;
2693}
2694
2695bool AArch64LegalizerInfo::legalizeConcatVectors(
2696 MachineInstr &MI, MachineRegisterInfo &MRI,
2697 MachineIRBuilder &MIRBuilder) const {
2698 // Widen sub-byte element vectors to byte-sized elements before concatenating.
2699 // This is analogous to SDAG's integer type promotion for sub-byte types.
2700 auto &Concat = cast<GConcatVectors>(Val&: MI);
2701 Register DstReg = Concat.getReg(Idx: 0);
2702 LLT DstTy = MRI.getType(Reg: DstReg);
2703 assert(DstTy.getScalarSizeInBits() < 8 && "Expected dst ty to be < 8b");
2704
2705 unsigned WideEltSize =
2706 std::max(a: 8u, b: (unsigned)PowerOf2Ceil(A: DstTy.getScalarSizeInBits()));
2707 LLT SrcTy = MRI.getType(Reg: Concat.getSourceReg(I: 0));
2708 LLT WideSrcTy = SrcTy.changeElementSize(NewEltSize: WideEltSize);
2709 LLT WideDstTy = DstTy.changeElementSize(NewEltSize: WideEltSize);
2710
2711 SmallVector<Register> WideSrcs;
2712 for (unsigned I = 0; I < Concat.getNumSources(); ++I) {
2713 auto Wide = MIRBuilder.buildAnyExt(Res: WideSrcTy, Op: Concat.getSourceReg(I));
2714 WideSrcs.push_back(Elt: Wide.getReg(Idx: 0));
2715 }
2716
2717 auto WideConcat = MIRBuilder.buildConcatVectors(Res: WideDstTy, Ops: WideSrcs);
2718 MIRBuilder.buildTrunc(Res: DstReg, Op: WideConcat);
2719 MI.eraseFromParent();
2720 return true;
2721}
2722
2723bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
2724 MachineIRBuilder &MIRBuilder,
2725 MachineRegisterInfo &MRI) const {
2726 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
2727
2728 // This function legalizes f64 -> bf16 and f64 -> f16 truncations via f64 ->
2729 // f32 G_FPTRUNC_ODD and f32 -> [b]f16 G_FPTRUNC, which apparently avoids the
2730 // usual double-rounding issue that could be present from using twin
2731 // G_FPTRUNC.
2732
2733 if (DstTy.isBFloat16() && SrcTy.isFloat64()) {
2734 auto Mid = MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {LLT::float32()},
2735 SrcOps: {Src}, Flags: MI.getFlags());
2736 MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC, DstOps: {Dst}, SrcOps: {Mid}, Flags: MI.getFlags());
2737 MI.eraseFromParent();
2738 return true;
2739 }
2740
2741 assert(SrcTy.isFixedVector() && isPowerOf2_32(SrcTy.getNumElements()) &&
2742 "Expected a power of 2 elements");
2743
2744 // We must mutate types here as FPTrunc may be used on a IEEE floating point
2745 // or a brainfloat.
2746 LLT v2s16 = DstTy.changeElementCount(NumElements: 2);
2747 LLT v4s16 = DstTy.changeElementCount(NumElements: 4);
2748 LLT v2s32 = SrcTy.changeElementCount(NumElements: 2).changeElementSize(NewEltSize: 32);
2749 LLT v4s32 = SrcTy.changeElementCount(NumElements: 4).changeElementSize(NewEltSize: 32);
2750 LLT v2s64 = SrcTy.changeElementCount(NumElements: 2);
2751
2752 SmallVector<Register> RegsToUnmergeTo;
2753 SmallVector<Register> TruncOddDstRegs;
2754 SmallVector<Register> RegsToMerge;
2755
2756 unsigned ElemCount = SrcTy.getNumElements();
2757
2758 // Find the biggest size chunks we can work with
2759 int StepSize = ElemCount % 4 ? 2 : 4;
2760
2761 // If we have a power of 2 greater than 2, we need to first unmerge into
2762 // enough pieces
2763 if (ElemCount <= 2)
2764 RegsToUnmergeTo.push_back(Elt: Src);
2765 else {
2766 for (unsigned i = 0; i < ElemCount / 2; ++i)
2767 RegsToUnmergeTo.push_back(Elt: MRI.createGenericVirtualRegister(Ty: v2s64));
2768
2769 MIRBuilder.buildUnmerge(Res: RegsToUnmergeTo, Op: Src);
2770 }
2771
2772 // Create all of the round-to-odd instructions and store them
2773 for (auto SrcReg : RegsToUnmergeTo) {
2774 Register Mid = MIRBuilder
2775 .buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {v2s32}, SrcOps: {SrcReg},
2776 Flags: MI.getFlags())
2777 .getReg(Idx: 0);
2778 TruncOddDstRegs.push_back(Elt: Mid);
2779 }
2780
2781 // Truncate 4s32 to 4s16 if we can to reduce instruction count, otherwise
2782 // truncate 2s32 to 2s16.
2783 unsigned Index = 0;
2784 for (unsigned LoopIter = 0; LoopIter < ElemCount / StepSize; ++LoopIter) {
2785 if (StepSize == 4) {
2786 Register ConcatDst =
2787 MIRBuilder
2788 .buildMergeLikeInstr(
2789 Res: {v4s32}, Ops: {TruncOddDstRegs[Index++], TruncOddDstRegs[Index++]})
2790 .getReg(Idx: 0);
2791
2792 RegsToMerge.push_back(
2793 Elt: MIRBuilder.buildFPTrunc(Res: v4s16, Op: ConcatDst, Flags: MI.getFlags()).getReg(Idx: 0));
2794 } else {
2795 RegsToMerge.push_back(
2796 Elt: MIRBuilder
2797 .buildFPTrunc(Res: v2s16, Op: TruncOddDstRegs[Index++], Flags: MI.getFlags())
2798 .getReg(Idx: 0));
2799 }
2800 }
2801
2802 // If there is only one register, replace the destination
2803 if (RegsToMerge.size() == 1) {
2804 MRI.replaceRegWith(FromReg: Dst, ToReg: RegsToMerge.pop_back_val());
2805 MI.eraseFromParent();
2806 return true;
2807 }
2808
2809 // Merge the rest of the instructions & replace the register
2810 Register Fin = MIRBuilder.buildMergeLikeInstr(Res: DstTy, Ops: RegsToMerge).getReg(Idx: 0);
2811 MRI.replaceRegWith(FromReg: Dst, ToReg: Fin);
2812 MI.eraseFromParent();
2813 return true;
2814}
2815
2816bool AArch64LegalizerInfo::legalizeGetRounding(MachineInstr &MI,
2817 MachineIRBuilder &MIRBuilder,
2818 MachineRegisterInfo &MRI,
2819 LegalizerHelper &Helper) const {
2820 const LLT I32 = LLT::integer(SizeInBits: 32);
2821 const LLT I64 = LLT::integer(SizeInBits: 64);
2822
2823 Register Dst = MI.getOperand(i: 0).getReg();
2824 Register FPCR64 = MRI.createGenericVirtualRegister(Ty: I64);
2825 MachineInstrBuilder GetFPCR =
2826 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_get_fpcr, Res: ArrayRef{FPCR64});
2827
2828 // AArch64 rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3,
2829 // 3->0, so we add one to the FPCR bits for the rounding mode.
2830 // Instead of shifting and then adding as `((FPCR >> 22) + 1) & 0b11` which
2831 // generates 3 instructions, we increment the rounding mode with
2832 // `(FPCR + (1 << 22))` and extract the bits. The shift and addition is done
2833 // in one instruction as `add .., .., #1024, lsl #12`, so overall we generate
2834 // one less instruction.
2835 auto FPCR32 = MIRBuilder.buildTrunc(Res: I32, Op: GetFPCR);
2836 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1U << 22);
2837 auto Added = MIRBuilder.buildAdd(Dst: I32, Src0: FPCR32, Src1: One);
2838 auto LSB = MIRBuilder.buildConstant(Res: I32, Val: 22);
2839 auto Width = MIRBuilder.buildConstant(Res: I32, Val: 2);
2840 MIRBuilder.buildInstr(Opc: TargetOpcode::G_UBFX, DstOps: {Dst}, SrcOps: {Added, LSB, Width});
2841
2842 MI.eraseFromParent();
2843 return true;
2844}
2845
2846bool AArch64LegalizerInfo::legalizeSetRounding(MachineInstr &MI,
2847 MachineIRBuilder &MIRBuilder,
2848 MachineRegisterInfo &MRI,
2849 LegalizerHelper &Helper) const {
2850 const LLT I32 = LLT::integer(SizeInBits: 32);
2851 const LLT I64 = LLT::integer(SizeInBits: 64);
2852
2853 // AArch64 rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3,
2854 // 3->0, so calculate the new value of FPCR[23:22] as `((arg - 1) & 3) << 22`.
2855 Register RM = MI.getOperand(i: 0).getReg();
2856 auto One = MIRBuilder.buildConstant(Res: I32, Val: 1);
2857 auto Subtracted = MIRBuilder.buildSub(Dst: I32, Src0: RM, Src1: One);
2858 auto Mask = MIRBuilder.buildConstant(Res: I32, Val: 0b11);
2859 auto Masked = MIRBuilder.buildAnd(Dst: I32, Src0: Subtracted, Src1: Mask);
2860 auto ShiftAmount = MIRBuilder.buildConstant(Res: I32, Val: 22);
2861 auto Shifted = MIRBuilder.buildShl(Dst: I32, Src0: Masked, Src1: ShiftAmount);
2862
2863 // Get current value of FPCR.
2864 MachineInstrBuilder GetFPCR =
2865 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_get_fpcr, Res: {I64});
2866
2867 // (FPCR & ~Mask) | Shifted
2868 auto FPCRMask = MIRBuilder.buildConstant(Res: I64, Val: ~((int64_t)0b11 << 22));
2869 auto FPCRMasked = MIRBuilder.buildAnd(Dst: I64, Src0: GetFPCR, Src1: FPCRMask);
2870 auto ShiftedS64 = MIRBuilder.buildZExt(Res: I64, Op: Shifted);
2871 auto FPCRUpdated = MIRBuilder.buildOr(Dst: I64, Src0: FPCRMasked, Src1: ShiftedS64);
2872
2873 // Write new FPCR.
2874 MIRBuilder.buildIntrinsic(ID: Intrinsic::aarch64_set_fpcr, Res: ArrayRef<Register>())
2875 .addUse(RegNo: FPCRUpdated.getReg(Idx: 0));
2876
2877 MI.eraseFromParent();
2878
2879 return true;
2880}
2881