1//===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for
10/// AArch64.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
14#include "AArch64LegalizerInfo.h"
15#include "AArch64Subtarget.h"
16#include "llvm/ADT/STLExtras.h"
17#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
18#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
20#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
22#include "llvm/CodeGen/GlobalISel/Utils.h"
23#include "llvm/CodeGen/MachineInstr.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/TargetOpcodes.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/Intrinsics.h"
29#include "llvm/IR/IntrinsicsAArch64.h"
30#include "llvm/IR/Type.h"
31#include "llvm/Support/MathExtras.h"
32#include <initializer_list>
33
34#define DEBUG_TYPE "aarch64-legalinfo"
35
36using namespace llvm;
37using namespace LegalizeActions;
38using namespace LegalizeMutations;
39using namespace LegalityPredicates;
40using namespace MIPatternMatch;
41
42AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
43 : ST(&ST) {
44 using namespace TargetOpcode;
45 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
46 const LLT s8 = LLT::scalar(SizeInBits: 8);
47 const LLT s16 = LLT::scalar(SizeInBits: 16);
48 const LLT s32 = LLT::scalar(SizeInBits: 32);
49 const LLT s64 = LLT::scalar(SizeInBits: 64);
50 const LLT s128 = LLT::scalar(SizeInBits: 128);
51 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
52 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
53 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
54 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
55 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
56 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
57 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
58 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
59 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
60 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
61 const LLT v2p0 = LLT::fixed_vector(NumElements: 2, ScalarTy: p0);
62
63 const LLT nxv16s8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: s8);
64 const LLT nxv8s16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: s16);
65 const LLT nxv4s32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: s32);
66 const LLT nxv2s64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: s64);
67
68 const LLT bf16 = LLT::bfloat16();
69 const LLT v4bf16 = LLT::fixed_vector(NumElements: 4, ScalarTy: bf16);
70 const LLT v8bf16 = LLT::fixed_vector(NumElements: 8, ScalarTy: bf16);
71
72 const LLT f16 = LLT::float16();
73 const LLT v4f16 = LLT::fixed_vector(NumElements: 4, ScalarTy: f16);
74 const LLT v8f16 = LLT::fixed_vector(NumElements: 8, ScalarTy: f16);
75
76 const LLT f32 = LLT::float32();
77 const LLT v2f32 = LLT::fixed_vector(NumElements: 2, ScalarTy: f32);
78 const LLT v4f32 = LLT::fixed_vector(NumElements: 4, ScalarTy: f32);
79
80 const LLT f64 = LLT::float64();
81 const LLT v2f64 = LLT::fixed_vector(NumElements: 2, ScalarTy: f64);
82
83 const LLT f128 = LLT::float128();
84
85 const LLT i8 = LLT::integer(SizeInBits: 8);
86 const LLT v8i8 = LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
87 const LLT v16i8 = LLT::fixed_vector(NumElements: 16, ScalarTy: i8);
88
89 const LLT i16 = LLT::integer(SizeInBits: 16);
90 const LLT v8i16 = LLT::fixed_vector(NumElements: 8, ScalarTy: i16);
91 const LLT v4i16 = LLT::fixed_vector(NumElements: 4, ScalarTy: i16);
92
93 const LLT i32 = LLT::integer(SizeInBits: 32);
94 const LLT v2i32 = LLT::fixed_vector(NumElements: 2, ScalarTy: i32);
95 const LLT v4i32 = LLT::fixed_vector(NumElements: 4, ScalarTy: i32);
96
97 const LLT i64 = LLT::integer(SizeInBits: 64);
98 const LLT v2i64 = LLT::fixed_vector(NumElements: 2, ScalarTy: i64);
99
100 const LLT i128 = LLT::integer(SizeInBits: 128);
101
102 const LLT nxv16i8 = LLT::scalable_vector(MinNumElements: 16, ScalarTy: i8);
103 const LLT nxv8i16 = LLT::scalable_vector(MinNumElements: 8, ScalarTy: i16);
104 const LLT nxv4i32 = LLT::scalable_vector(MinNumElements: 4, ScalarTy: i32);
105 const LLT nxv2i64 = LLT::scalable_vector(MinNumElements: 2, ScalarTy: i64);
106
107 std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
108 v16s8, v8s16, v4s32,
109 v2s64, v2p0,
110 /* End 128bit types */
111 /* Begin 64bit types */
112 v8s8, v4s16, v2s32};
113 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
114 SmallVector<LLT, 8> PackedVectorAllTypesVec(PackedVectorAllTypeList);
115 SmallVector<LLT, 8> ScalarAndPtrTypesVec(ScalarAndPtrTypesList);
116
117 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
118
119 // FIXME: support subtargets which have neon/fp-armv8 disabled.
120 if (!ST.hasNEON() || !ST.hasFPARMv8())
121 return;
122
123 // Some instructions only support s16 if the subtarget has full 16-bit FP
124 // support.
125 const bool HasFP16 = ST.hasFullFP16();
126 const bool HasCSSC = ST.hasCSSC();
127 const bool HasRCPC3 = ST.hasRCPC3();
128 const bool HasSVE = ST.hasSVE();
129
130 getActionDefinitionsBuilder(
131 Opcodes: {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
132 .legalFor(Types: {p0, s8, s16, s32, s64, s128})
133 .legalFor(Types: {v2s8, v4s8, v8s8, v16s8, v2s16, v4s16, v8s16, v2s32, v4s32,
134 v2s64, v2p0})
135 .widenScalarToNextPow2(TypeIdx: 0)
136 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
137 .moreElementsToNextPow2(TypeIdx: 0)
138 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
139 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
140 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
141 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
142 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
143 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
144 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
145
146 getActionDefinitionsBuilder(Opcode: G_PHI)
147 .legalFor(Types: {p0, s16, s32, s64})
148 .legalFor(Types: PackedVectorAllTypeList)
149 .widenScalarToNextPow2(TypeIdx: 0)
150 .moreElementsToNextPow2(TypeIdx: 0)
151 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
152 .clampScalar(TypeIdx: 0, MinTy: s16, MaxTy: s64)
153 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
154 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
155 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
156 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
157 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
158 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8);
159
160 getActionDefinitionsBuilder(Opcode: G_INSERT)
161 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64, p0}), P1: typeInSet(TypeIdx: 1, TypesInit: {s8, s16, s32}),
162 args: smallerThan(TypeIdx0: 1, TypeIdx1: 0)))
163 .widenScalarToNextPow2(TypeIdx: 0)
164 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
165 .widenScalarToNextPow2(TypeIdx: 1)
166 .minScalar(TypeIdx: 1, Ty: s8)
167 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s32}), TypeIdx: 1, Ty: s16)
168 .maxScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {s64, p0}), TypeIdx: 1, Ty: s32);
169
170 getActionDefinitionsBuilder(Opcode: G_EXTRACT)
171 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64, p0}),
172 P1: typeInSet(TypeIdx: 1, TypesInit: {s32, s64, s128, p0}), args: smallerThan(TypeIdx0: 0, TypeIdx1: 1)))
173 .widenScalarToNextPow2(TypeIdx: 1)
174 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s128)
175 .widenScalarToNextPow2(TypeIdx: 0)
176 .minScalar(TypeIdx: 0, Ty: s16)
177 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s32}), TypeIdx: 0, Ty: s16)
178 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s64, p0}), TypeIdx: 0, Ty: s32)
179 .maxScalarIf(Predicate: typeInSet(TypeIdx: 1, TypesInit: {s128}), TypeIdx: 0, Ty: s64);
180
181 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB, G_AND, G_OR, G_XOR})
182 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
183 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
184 .widenScalarToNextPow2(TypeIdx: 0)
185 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
186 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
187 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
188 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
189 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
190 .minScalarOrEltIf(
191 Predicate: [=](const LegalityQuery &Query) {
192 return Query.Types[0].getNumElements() <= 2;
193 },
194 TypeIdx: 0, Ty: s32)
195 .minScalarOrEltIf(
196 Predicate: [=](const LegalityQuery &Query) {
197 return Query.Types[0].getNumElements() <= 4;
198 },
199 TypeIdx: 0, Ty: s16)
200 .minScalarOrEltIf(
201 Predicate: [=](const LegalityQuery &Query) {
202 return Query.Types[0].getNumElements() <= 16;
203 },
204 TypeIdx: 0, Ty: s8)
205 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
206 .moreElementsToNextPow2(TypeIdx: 0);
207
208 getActionDefinitionsBuilder(Opcode: G_MUL)
209 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
210 .widenScalarToNextPow2(TypeIdx: 0)
211 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
212 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
213 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
214 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
215 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
216 .minScalarOrEltIf(
217 Predicate: [=](const LegalityQuery &Query) {
218 return Query.Types[0].getNumElements() <= 2;
219 },
220 TypeIdx: 0, Ty: s32)
221 .minScalarOrEltIf(
222 Predicate: [=](const LegalityQuery &Query) {
223 return Query.Types[0].getNumElements() <= 4;
224 },
225 TypeIdx: 0, Ty: s16)
226 .minScalarOrEltIf(
227 Predicate: [=](const LegalityQuery &Query) {
228 return Query.Types[0].getNumElements() <= 16;
229 },
230 TypeIdx: 0, Ty: s8)
231 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
232 .moreElementsToNextPow2(TypeIdx: 0);
233
234 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_ASHR, G_LSHR})
235 .customIf(Predicate: [=](const LegalityQuery &Query) {
236 const auto &SrcTy = Query.Types[0];
237 const auto &AmtTy = Query.Types[1];
238 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
239 AmtTy.getSizeInBits() == 32;
240 })
241 .legalFor(Types: {
242 {i32, i32},
243 {i32, i64},
244 {i64, i64},
245 {v8i8, v8i8},
246 {v16i8, v16i8},
247 {v4i16, v4i16},
248 {v8i16, v8i16},
249 {v2i32, v2i32},
250 {v4i32, v4i32},
251 {v2i64, v2i64},
252 })
253 .widenScalarToNextPow2(TypeIdx: 1)
254 .widenScalarToNextPow2(TypeIdx: 0)
255 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
256 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
257 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
258 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
259 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
260 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
261 .moreElementsToNextPow2(TypeIdx: 0)
262 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0)
263 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
264 .minScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, LargeTypeIdx: 0)
265 .maxScalarEltSameAsIf(Predicate: isVector(TypeIdx: 0), TypeIdx: 1, SmallTypeIdx: 0);
266
267 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
268 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
269 .clampScalarOrElt(TypeIdx: 1, MinTy: s64, MaxTy: s64)
270 .clampNumElements(TypeIdx: 0, MinTy: v2p0, MaxTy: v2p0);
271
272 getActionDefinitionsBuilder(Opcode: G_PTRMASK).legalFor(Types: {{p0, s64}});
273
274 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_UDIV})
275 .legalFor(Types: {i32, i64})
276 .libcallFor(Types: {i128})
277 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
278 .widenScalarToNextPow2(TypeIdx: 0)
279 .scalarize(TypeIdx: 0);
280
281 getActionDefinitionsBuilder(Opcodes: {G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
282 .lowerFor(Types: {i8, i16, i32, i64, v2i32, v4i32, v2i64})
283 .libcallFor(Types: {i128})
284 .widenScalarOrEltToNextPow2(TypeIdx: 0)
285 .minScalarOrElt(TypeIdx: 0, Ty: s32)
286 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
287 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
288 .scalarize(TypeIdx: 0);
289
290 getActionDefinitionsBuilder(Opcodes: {G_SMULO, G_UMULO})
291 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
292 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
293 .lower();
294
295 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
296 .legalFor(Types: {i64, v16i8, v8i16, v4i32})
297 .lower();
298
299 getActionDefinitionsBuilder(Opcodes: {G_SMULFIX, G_UMULFIX}).lower();
300
301 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX})
302 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
303 .legalFor(Pred: HasCSSC, Types: {i32, i64})
304 .minScalar(Pred: HasCSSC, TypeIdx: 0, Ty: s32)
305 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
306 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
307 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
308 .lower();
309
310 // FIXME: Legal vector types are only legal with NEON.
311 getActionDefinitionsBuilder(Opcode: G_ABS)
312 .legalFor(Pred: HasCSSC, Types: {i32, i64})
313 .legalFor(Types: {v16i8, v8i16, v4i32, v2i64, v2p0, v8i8, v4i16, v2i32})
314 .customIf(Predicate: [=](const LegalityQuery &Q) {
315 // TODO: Fix suboptimal codegen for 128+ bit types.
316 LLT SrcTy = Q.Types[0];
317 return SrcTy.isScalar() && SrcTy.getSizeInBits() < 128;
318 })
319 .widenScalarIf(
320 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v4s8; },
321 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v4i16); })
322 .widenScalarIf(
323 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0] == v2s16; },
324 Mutation: [=](const LegalityQuery &Query) { return std::make_pair(x: 0, y: v2i32); })
325 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
326 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
327 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
328 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
329 .moreElementsToNextPow2(TypeIdx: 0)
330 .lower();
331
332 getActionDefinitionsBuilder(
333 Opcodes: {G_ABDS, G_ABDU, G_UAVGFLOOR, G_UAVGCEIL, G_SAVGFLOOR, G_SAVGCEIL})
334 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
335 .lower();
336
337 getActionDefinitionsBuilder(
338 Opcodes: {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
339 .legalFor(Types: {{i32, i32}, {i64, i32}})
340 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
341 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
342 .widenScalarToNextPow2(TypeIdx: 0);
343
344 getActionDefinitionsBuilder(Opcodes: {G_FSHL, G_FSHR})
345 .customFor(Types: {{i32, i32}, {i32, i64}, {i64, i64}})
346 .lower();
347
348 getActionDefinitionsBuilder(Opcode: G_ROTR)
349 .legalFor(Types: {{i32, i64}, {i64, i64}})
350 .customIf(Predicate: [=](const LegalityQuery &Q) {
351 return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
352 })
353 .lower();
354 getActionDefinitionsBuilder(Opcode: G_ROTL).lower();
355
356 getActionDefinitionsBuilder(Opcodes: {G_SBFX, G_UBFX})
357 .customFor(Types: {{s32, s32}, {s64, s64}});
358
359 auto always = [=](const LegalityQuery &Q) { return true; };
360 getActionDefinitionsBuilder(Opcode: G_CTPOP)
361 .legalFor(Pred: HasCSSC, Types: {{i32, i32}, {i64, i64}})
362 .legalFor(Types: {{v8i8, v8i8}, {v16i8, v16i8}})
363 .customFor(Pred: !HasCSSC, Types: {{s32, s32}, {s64, s64}})
364 .customFor(Types: {{s128, s128},
365 {v4s16, v4s16},
366 {v8s16, v8s16},
367 {v2s32, v2s32},
368 {v4s32, v4s32},
369 {v2s64, v2s64}})
370 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s128)
371 .widenScalarToNextPow2(TypeIdx: 0)
372 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
373 .minScalarEltSameAsIf(Predicate: always, TypeIdx: 1, LargeTypeIdx: 0)
374 .maxScalarEltSameAsIf(Predicate: always, TypeIdx: 1, SmallTypeIdx: 0)
375 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
376 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
377 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
378 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
379 .moreElementsToNextPow2(TypeIdx: 0)
380 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
381
382 getActionDefinitionsBuilder(Opcodes: {G_CTLZ, G_CTLS})
383 .legalFor(Types: {{i32, i32},
384 {i64, i64},
385 {v8i8, v8i8},
386 {v16i8, v16i8},
387 {v4i16, v4i16},
388 {v8i16, v8i16},
389 {v2i32, v2i32},
390 {v4i32, v4i32}})
391 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
392 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
393 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1, /*Min=*/MinSize: 8)
394 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
395 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
396 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
397 .moreElementsToNextPow2(TypeIdx: 0)
398 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 32), TypeIdx: 0)
399 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
400
401 getActionDefinitionsBuilder(Opcode: G_INSERT_SUBVECTOR).lower();
402
403 getActionDefinitionsBuilder(Opcode: G_CTLZ_ZERO_POISON).lower();
404
405 getActionDefinitionsBuilder(Opcode: G_CTTZ)
406 .lowerIf(Predicate: isVector(TypeIdx: 0))
407 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 32)
408 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
409 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1)
410 .legalFor(Pred: HasCSSC, Types: {s32, s64})
411 .customFor(Pred: !HasCSSC, Types: {s32, s64});
412
413 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON).lower();
414
415 getActionDefinitionsBuilder(Opcode: G_BITREVERSE)
416 .legalFor(Types: {i32, i64, v8i8, v16i8})
417 .widenScalarToNextPow2(TypeIdx: 0, /*Min = */ MinSize: 32)
418 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
419 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
420 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
421 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
422 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
423 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
424 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
425 .moreElementsToNextPow2(TypeIdx: 0)
426 .lower();
427
428 getActionDefinitionsBuilder(Opcode: G_CLMUL).legalFor(Types: {v8i8, v16i8});
429
430 getActionDefinitionsBuilder(Opcode: G_BSWAP)
431 .legalFor(Types: {i32, i64, v4i16, v8i16, v2i32, v4i32, v2i64})
432 .widenScalarOrEltToNextPow2(TypeIdx: 0, MinSize: 16)
433 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
434 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
435 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
436 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
437 .moreElementsToNextPow2(TypeIdx: 0);
438
439 getActionDefinitionsBuilder(Opcodes: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT})
440 .legalFor(Types: {v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
441 .legalFor(Pred: HasSVE, Types: {nxv16i8, nxv8i16, nxv4i32, nxv2i64})
442 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
443 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
444 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
445 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
446 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
447 .moreElementsToNextPow2(TypeIdx: 0)
448 .lower();
449
450 getActionDefinitionsBuilder(
451 Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
452 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
453 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
454 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
455 .legalFor(Pred: HasFP16, Types: {f16, v4f16, v8f16})
456 .libcallFor(Types: {f128})
457 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
458 .widenScalarIf(
459 Predicate: [=](const LegalityQuery &Q) {
460 return (!HasFP16 && Q.Types[0].getScalarType().isFloat16()) ||
461 Q.Types[0].getScalarType().isBFloat16();
462 },
463 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
464 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
465 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
466 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
467 .moreElementsToNextPow2(TypeIdx: 0);
468
469 getActionDefinitionsBuilder(Opcodes: {G_FABS, G_FNEG})
470 .legalFor(Types: {f32, f64, v2f32, v4f32, v2f64})
471 .legalFor(Pred: HasFP16, Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16})
472 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
473 .lowerIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64))
474 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
475 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
476 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
477 .moreElementsToNextPow2(TypeIdx: 0)
478 .lowerFor(Types: {f16, bf16, v4f16, v4bf16, v8f16, v8bf16});
479
480 getActionDefinitionsBuilder(Opcodes: {G_FREM, G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2,
481 G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10,
482 G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH,
483 G_FSINH, G_FTANH, G_FMODF})
484 .libcallFor(Types: {f32, f64, f128})
485 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
486 .scalarize(TypeIdx: 0);
487 getActionDefinitionsBuilder(Opcodes: {G_FPOWI, G_FLDEXP})
488 .libcallFor(Types: {{f32, i32}, {f64, i32}, {f128, i32}})
489 .widenScalarFor(Types: {f16, bf16}, Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
490 .scalarize(TypeIdx: 0);
491
492 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_INTRINSIC_LRINT})
493 .legalFor(Types: {{i32, f32}, {i32, f64}, {i64, f32}, {i64, f64}})
494 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {i64, f16}})
495 .minScalar(TypeIdx: 1, Ty: s32)
496 .libcallFor(Types: {{s64, s128}})
497 .lower();
498 getActionDefinitionsBuilder(Opcodes: {G_LLROUND, G_INTRINSIC_LLRINT})
499 .legalFor(Types: {{i64, f32}, {i64, f64}})
500 .legalFor(Pred: HasFP16, Types: {{i64, f16}})
501 .minScalar(TypeIdx: 0, Ty: s64)
502 .minScalar(TypeIdx: 1, Ty: s32)
503 .libcallFor(Types: {{s64, s128}})
504 .lower();
505
506 // TODO: Custom legalization for mismatched types.
507 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
508 .moreElementsIf(
509 Predicate: [](const LegalityQuery &Query) { return Query.Types[0].isScalar(); },
510 Mutation: [=](const LegalityQuery &Query) {
511 const LLT Ty = Query.Types[0];
512 return std::pair(0, LLT::fixed_vector(NumElements: Ty == s16 ? 4 : 2, ScalarTy: Ty));
513 })
514 .lower();
515
516 getActionDefinitionsBuilder(Opcode: G_FMAD).lower();
517
518 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
519 auto &Actions = getActionDefinitionsBuilder(Opcode: Op);
520
521 if (Op == G_SEXTLOAD)
522 Actions.lowerIf(Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered));
523
524 // Atomics have zero extending behavior.
525 Actions
526 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8},
527 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8},
528 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
529 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 2},
530 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 2},
531 {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 4},
532 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
533 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
534 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}})
535 .widenScalarToNextPow2(TypeIdx: 0)
536 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
537 // TODO: We could support sum-of-pow2's but the lowering code doesn't know
538 // how to do that yet.
539 .unsupportedIfMemSizeNotPow2()
540 // Lower anything left over into G_*EXT and G_LOAD
541 .lower();
542 }
543
544 auto IsPtrVecPred = [=](const LegalityQuery &Query) {
545 const LLT &ValTy = Query.Types[0];
546 return ValTy.isPointerVector() && ValTy.getAddressSpace() == 0;
547 };
548
549 getActionDefinitionsBuilder(Opcode: G_LOAD)
550 .customIf(Predicate: [=](const LegalityQuery &Query) {
551 return HasRCPC3 && Query.Types[0] == s128 &&
552 Query.MMODescrs[0].Ordering == AtomicOrdering::Acquire;
553 })
554 .customIf(Predicate: [=](const LegalityQuery &Query) {
555 return Query.Types[0] == s128 &&
556 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
557 })
558 .legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8},
559 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8},
560 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
561 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8},
562 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8},
563 {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8},
564 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8},
565 {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
566 {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8},
567 {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
568 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8},
569 {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8},
570 {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
571 // These extends are also legal
572 .legalForTypesWithMemDesc(
573 TypesAndMemDesc: {{.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}})
574 .legalForTypesWithMemDesc(TypesAndMemDesc: {
575 // SVE vscale x 128 bit base sizes
576 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
577 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
578 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
579 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
580 })
581 .widenScalarToNextPow2(TypeIdx: 0, /* MinSize = */ 8)
582 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
583 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
584 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
585 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
586 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
587 .lowerIfMemSizeNotByteSizePow2()
588 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
589 .narrowScalarIf(
590 Predicate: [=](const LegalityQuery &Query) {
591 // Clamp extending load results to 32-bits.
592 return Query.Types[0].isScalar() &&
593 Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
594 Query.Types[0].getSizeInBits() > 32;
595 },
596 Mutation: changeTo(TypeIdx: 0, Ty: s32))
597 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
598 .bitcastIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
599 Mutation: [=](const LegalityQuery &Query) {
600 const LLT VecTy = Query.Types[0];
601 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
602 })
603 .customIf(Predicate: IsPtrVecPred)
604 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
605 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
606
607 getActionDefinitionsBuilder(Opcode: G_STORE)
608 .customIf(Predicate: [=](const LegalityQuery &Query) {
609 return HasRCPC3 && Query.Types[0] == s128 &&
610 Query.MMODescrs[0].Ordering == AtomicOrdering::Release;
611 })
612 .customIf(Predicate: [=](const LegalityQuery &Query) {
613 return Query.Types[0] == s128 &&
614 Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
615 })
616 .widenScalarIf(
617 Predicate: all(P0: scalarNarrowerThan(TypeIdx: 0, Size: 32),
618 P1: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Release)),
619 Mutation: changeTo(TypeIdx: 0, Ty: s32))
620 .legalForTypesWithMemDesc(
621 TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s16
622 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s32
623 {.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 8}, // truncstorei8 from s64
624 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s32
625 {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 8}, // truncstorei16 from s64
626 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 8}, {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 8},
627 {.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 8}, // truncstorei32 from s64
628 {.Type0: p0, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: s128, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v16s8, .Type1: p0, .MemTy: s128, .Align: 8},
629 {.Type0: v8s8, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s16, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v8s16, .Type1: p0, .MemTy: s128, .Align: 8},
630 {.Type0: v2s32, .Type1: p0, .MemTy: s64, .Align: 8}, {.Type0: v4s32, .Type1: p0, .MemTy: s128, .Align: 8}, {.Type0: v2s64, .Type1: p0, .MemTy: s128, .Align: 8}})
631 .legalForTypesWithMemDesc(TypesAndMemDesc: {
632 // SVE vscale x 128 bit base sizes
633 // TODO: Add nxv2p0. Consider bitcastIf.
634 // See #92130
635 // https://github.com/llvm/llvm-project/pull/92130#discussion_r1616888461
636 {.Type0: nxv16s8, .Type1: p0, .MemTy: nxv16s8, .Align: 8},
637 {.Type0: nxv8s16, .Type1: p0, .MemTy: nxv8s16, .Align: 8},
638 {.Type0: nxv4s32, .Type1: p0, .MemTy: nxv4s32, .Align: 8},
639 {.Type0: nxv2s64, .Type1: p0, .MemTy: nxv2s64, .Align: 8},
640 })
641 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
642 .minScalarOrElt(TypeIdx: 0, Ty: s8)
643 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
644 return Query.Types[0].isScalar() &&
645 Query.Types[0] != Query.MMODescrs[0].MemoryTy;
646 })
647 // Maximum: sN * k = 128
648 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 16)
649 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 8)
650 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 4)
651 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
652 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
653 .lowerIfMemSizeNotPow2()
654 // TODO: Use BITCAST for v2i8, v2i16 after G_TRUNC gets sorted out
655 .bitcastIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {v4s8}),
656 P1: LegalityPredicate([=](const LegalityQuery &Query) {
657 return Query.Types[0].getSizeInBits() ==
658 Query.MMODescrs[0].MemoryTy.getSizeInBits();
659 })),
660 Mutation: [=](const LegalityQuery &Query) {
661 const LLT VecTy = Query.Types[0];
662 return std::pair(0, LLT::integer(SizeInBits: VecTy.getSizeInBits()));
663 })
664 .customIf(Predicate: IsPtrVecPred)
665 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: {v2s16, v2s8}), TypeIdx: 0)
666 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
667 .lower();
668
669 getActionDefinitionsBuilder(Opcode: G_INDEXED_STORE)
670 // Idx 0 == Ptr, Idx 1 == Val
671 // TODO: we can implement legalizations but as of now these are
672 // generated in a very specific way.
673 .legalForTypesWithMemDesc(TypesAndMemDesc: {
674 {.Type0: p0, .Type1: s8, .MemTy: s8, .Align: 8},
675 {.Type0: p0, .Type1: s16, .MemTy: s16, .Align: 8},
676 {.Type0: p0, .Type1: s32, .MemTy: s8, .Align: 8},
677 {.Type0: p0, .Type1: s32, .MemTy: s16, .Align: 8},
678 {.Type0: p0, .Type1: s32, .MemTy: s32, .Align: 8},
679 {.Type0: p0, .Type1: s64, .MemTy: s64, .Align: 8},
680 {.Type0: p0, .Type1: p0, .MemTy: p0, .Align: 8},
681 {.Type0: p0, .Type1: v8s8, .MemTy: v8s8, .Align: 8},
682 {.Type0: p0, .Type1: v16s8, .MemTy: v16s8, .Align: 8},
683 {.Type0: p0, .Type1: v4s16, .MemTy: v4s16, .Align: 8},
684 {.Type0: p0, .Type1: v8s16, .MemTy: v8s16, .Align: 8},
685 {.Type0: p0, .Type1: v2s32, .MemTy: v2s32, .Align: 8},
686 {.Type0: p0, .Type1: v4s32, .MemTy: v4s32, .Align: 8},
687 {.Type0: p0, .Type1: v2s64, .MemTy: v2s64, .Align: 8},
688 {.Type0: p0, .Type1: v2p0, .MemTy: v2p0, .Align: 8},
689 {.Type0: p0, .Type1: s128, .MemTy: s128, .Align: 8},
690 })
691 .unsupported();
692
693 auto IndexedLoadBasicPred = [=](const LegalityQuery &Query) {
694 LLT LdTy = Query.Types[0];
695 LLT PtrTy = Query.Types[1];
696 if (!llvm::is_contained(Range: PackedVectorAllTypesVec, Element: LdTy) &&
697 !llvm::is_contained(Range: ScalarAndPtrTypesVec, Element: LdTy) && LdTy != s128)
698 return false;
699 if (PtrTy != p0)
700 return false;
701 return true;
702 };
703 getActionDefinitionsBuilder(Opcode: G_INDEXED_LOAD)
704 .unsupportedIf(
705 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
706 .legalIf(Predicate: IndexedLoadBasicPred)
707 .unsupported();
708 getActionDefinitionsBuilder(Opcodes: {G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD})
709 .unsupportedIf(
710 Predicate: atomicOrderingAtLeastOrStrongerThan(MMOIdx: 0, Ordering: AtomicOrdering::Unordered))
711 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s16, s32, s64}),
712 P1: LegalityPredicate([=](const LegalityQuery &Q) {
713 LLT LdTy = Q.Types[0];
714 LLT PtrTy = Q.Types[1];
715 LLT MemTy = Q.MMODescrs[0].MemoryTy;
716 if (PtrTy != p0)
717 return false;
718 if (LdTy == s16)
719 return MemTy == s8;
720 if (LdTy == s32)
721 return MemTy == s8 || MemTy == s16;
722 if (LdTy == s64)
723 return MemTy == s8 || MemTy == s16 || MemTy == s32;
724 return false;
725 })))
726 .unsupported();
727
728 // Constants
729 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
730 .legalFor(Types: {p0, s8, s16, s32, s64})
731 .widenScalarToNextPow2(TypeIdx: 0)
732 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64);
733 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
734 .legalFor(Types: {s16, s32, s64, s128});
735
736 // FIXME: fix moreElementsToNextPow2
737 getActionDefinitionsBuilder(Opcode: G_ICMP)
738 .legalFor(Types: {{i32, i32}, {i32, i64}, {i32, p0}})
739 .widenScalarOrEltToNextPow2(TypeIdx: 1)
740 .minScalarOrElt(TypeIdx: 1, Ty: s8)
741 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s64)
742 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
743 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
744 .minScalarEltSameAsIf(
745 Predicate: [=](const LegalityQuery &Query) {
746 const LLT &Ty = Query.Types[0];
747 const LLT &SrcTy = Query.Types[1];
748 return Ty.isVector() && !SrcTy.isPointerVector() &&
749 Ty.getElementType() != SrcTy.getElementType();
750 },
751 TypeIdx: 0, LargeTypeIdx: 1)
752 .minScalarOrEltIf(
753 Predicate: [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
754 TypeIdx: 1, Ty: s32)
755 .minScalarOrEltIf(
756 Predicate: [=](const LegalityQuery &Query) {
757 return Query.Types[1].isPointerVector();
758 },
759 TypeIdx: 0, Ty: s64)
760 .moreElementsToNextPow2(TypeIdx: 1)
761 .clampNumElements(TypeIdx: 1, MinTy: v8s8, MaxTy: v16s8)
762 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
763 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
764 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
765 .clampNumElements(TypeIdx: 1, MinTy: v2p0, MaxTy: v2p0)
766 .customIf(Predicate: isVector(TypeIdx: 0));
767
768 getActionDefinitionsBuilder(Opcode: G_FCMP)
769 .legalFor(Types: {{i32, f32},
770 {i32, f64},
771 {v4i32, v4f32},
772 {v2i32, v2f32},
773 {v2i64, v2f64}})
774 .legalFor(Pred: HasFP16, Types: {{i32, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
775 .widenScalarOrEltToNextPow2(TypeIdx: 1)
776 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32)
777 .widenScalarIf(
778 Predicate: [=](const LegalityQuery &Q) {
779 return (!HasFP16 && Q.Types[1].getScalarType().isFloat16()) ||
780 Q.Types[1].getScalarType().isBFloat16();
781 },
782 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
783 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
784 .minScalarEltSameAsIf(
785 Predicate: [=](const LegalityQuery &Query) {
786 const LLT &Ty = Query.Types[0];
787 const LLT &SrcTy = Query.Types[1];
788 return Ty.isVector() && !SrcTy.isPointerVector() &&
789 Ty.getElementType() != SrcTy.getElementType();
790 },
791 TypeIdx: 0, LargeTypeIdx: 1)
792 .clampNumElements(TypeIdx: 1, MinTy: v4s16, MaxTy: v8s16)
793 .clampNumElements(TypeIdx: 1, MinTy: v2s32, MaxTy: v4s32)
794 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
795 .moreElementsToNextPow2(TypeIdx: 1)
796 .libcallFor(Types: {{s32, s128}});
797
798 // Extensions
799 auto ExtLegalFunc = [=](const LegalityQuery &Query) {
800 unsigned DstSize = Query.Types[0].getSizeInBits();
801
802 // Handle legal vectors using legalFor
803 if (Query.Types[0].isVector())
804 return false;
805
806 if (DstSize < 8 || DstSize >= 128 || !isPowerOf2_32(Value: DstSize))
807 return false; // Extending to a scalar s128 needs narrowing.
808
809 const LLT &SrcTy = Query.Types[1];
810
811 // Make sure we fit in a register otherwise. Don't bother checking that
812 // the source type is below 128 bits. We shouldn't be allowing anything
813 // through which is wider than the destination in the first place.
814 unsigned SrcSize = SrcTy.getSizeInBits();
815 if (SrcSize < 8 || !isPowerOf2_32(Value: SrcSize))
816 return false;
817
818 return true;
819 };
820 getActionDefinitionsBuilder(Opcodes: {G_ZEXT, G_SEXT, G_ANYEXT})
821 .legalIf(Predicate: ExtLegalFunc)
822 .legalFor(Types: {{v8s16, v8s8}, {v4s32, v4s16}, {v2s64, v2s32}})
823 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64) // Just for s128, others are handled above.
824 .moreElementsToNextPow2(TypeIdx: 0)
825 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
826 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
827 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
828 // Tries to convert a large EXTEND into two smaller EXTENDs
829 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
830 return (Query.Types[0].getScalarSizeInBits() >
831 Query.Types[1].getScalarSizeInBits() * 2) &&
832 Query.Types[0].isVector() &&
833 (Query.Types[1].getScalarSizeInBits() == 8 ||
834 Query.Types[1].getScalarSizeInBits() == 16);
835 })
836 .clampMinNumElements(TypeIdx: 1, EltTy: s8, MinElements: 8)
837 .clampMinNumElements(TypeIdx: 1, EltTy: s16, MinElements: 4)
838 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
839
840 getActionDefinitionsBuilder(Opcode: G_TRUNC)
841 .legalFor(Types: {{v8s8, v8s16}, {v4s16, v4s32}, {v2s32, v2s64}})
842 .moreElementsToNextPow2(TypeIdx: 0)
843 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: 8)
844 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: 4)
845 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: 2)
846 .minScalarOrEltIf(
847 Predicate: [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
848 TypeIdx: 0, Ty: s8)
849 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
850 LLT DstTy = Query.Types[0];
851 LLT SrcTy = Query.Types[1];
852 return DstTy.isVector() && SrcTy.getSizeInBits() > 128 &&
853 DstTy.getScalarSizeInBits() * 2 <= SrcTy.getScalarSizeInBits();
854 })
855 .clampMinNumElements(TypeIdx: 0, EltTy: s8, MinElements: 8)
856 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 4)
857 .alwaysLegal();
858
859 getActionDefinitionsBuilder(Opcodes: {G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
860 .legalFor(Types: {{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
861 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v2s32);
862
863 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
864 .legalFor(Types: {i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
865 .maxScalar(TypeIdx: 0, Ty: s64)
866 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
867 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
868 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
869 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
870 .lower();
871
872 // FP conversions
873 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
874 .legalFor(
875 Types: {{f16, f32}, {f16, f64}, {f32, f64}, {v4f16, v4f32}, {v2f32, v2f64}})
876 .legalFor(Pred: ST.hasBF16(), Types: {{bf16, f32}, {v4bf16, v4f32}})
877 .libcallFor(Types: {{f16, f128}, {f32, f128}, {f64, f128}})
878 .moreElementsToNextPow2(TypeIdx: 1)
879 .customIf(Predicate: [](const LegalityQuery &Q) {
880 LLT DstTy = Q.Types[0];
881 LLT SrcTy = Q.Types[1];
882 return SrcTy.getScalarSizeInBits() == 64 &&
883 DstTy.getScalarSizeInBits() == 16;
884 })
885 .lowerFor(Types: {{bf16, f32}, {v4bf16, v4f32}})
886 // Clamp based on input
887 .clampNumElements(TypeIdx: 1, MinTy: v4s32, MaxTy: v4s32)
888 .clampNumElements(TypeIdx: 1, MinTy: v2s64, MaxTy: v2s64)
889 .scalarize(TypeIdx: 0);
890
891 getActionDefinitionsBuilder(Opcode: G_FPEXT)
892 .legalFor(Types: {{f32, f16},
893 {f64, f16},
894 {f32, bf16},
895 {f64, f32},
896 {v4f32, v4f16},
897 {v4f32, v4bf16},
898 {v2f64, v2f32}})
899 .libcallFor(Types: {{f128, f64}, {f128, f32}, {f128, f16}})
900 .moreElementsToNextPow2(TypeIdx: 0)
901 .widenScalarIf(
902 Predicate: [](const LegalityQuery &Q) {
903 LLT DstTy = Q.Types[0];
904 LLT SrcTy = Q.Types[1];
905 return SrcTy.isVector() && DstTy.isVector() &&
906 SrcTy.getScalarSizeInBits() == 16 &&
907 DstTy.getScalarSizeInBits() == 64;
908 },
909 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
910 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
911 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
912 .scalarize(TypeIdx: 0);
913
914 // Conversions
915 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
916 .legalFor(Types: {{i32, f32},
917 {i64, f32},
918 {i32, f64},
919 {i64, f64},
920 {v2i32, v2f32},
921 {v4i32, v4f32},
922 {v2i64, v2f64}})
923 .legalFor(Pred: HasFP16,
924 Types: {{i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
925 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
926 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
927 // The range of a fp16 value fits into an i17, so we can lower the width
928 // to i64.
929 .narrowScalarIf(
930 Predicate: [=](const LegalityQuery &Query) {
931 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
932 },
933 Mutation: changeTo(TypeIdx: 0, Ty: i64))
934 .moreElementsToNextPow2(TypeIdx: 0)
935 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0)
936 .minScalar(TypeIdx: 0, Ty: s32)
937 .widenScalarIf(
938 Predicate: [HasFP16](const LegalityQuery &Query) {
939 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
940 Query.Types[1].getScalarType().isBFloat16();
941 },
942 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
943 .widenScalarIf(
944 Predicate: [=](const LegalityQuery &Query) {
945 return Query.Types[0].getScalarSizeInBits() <= 64 &&
946 Query.Types[0].getScalarSizeInBits() >
947 Query.Types[1].getScalarSizeInBits();
948 },
949 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
950 .widenScalarIf(
951 Predicate: [=](const LegalityQuery &Query) {
952 return Query.Types[1].getScalarSizeInBits() <= 64 &&
953 Query.Types[0].getScalarSizeInBits() <
954 Query.Types[1].getScalarSizeInBits();
955 },
956 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
957 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
958 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
959 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
960 .libcallFor(
961 Types: {{i32, f128}, {i64, f128}, {i128, f128}, {i128, f32}, {i128, f64}});
962
963 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
964 .legalFor(Types: {{i32, f32},
965 {i64, f32},
966 {i32, f64},
967 {i64, f64},
968 {v2i32, v2f32},
969 {v4i32, v4f32},
970 {v2i64, v2f64}})
971 .legalFor(
972 Pred: HasFP16,
973 Types: {{i16, f16}, {i32, f16}, {i64, f16}, {v4i16, v4f16}, {v8i16, v8f16}})
974 // Handle types larger than i64 by scalarizing/lowering.
975 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
976 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
977 // The range of a fp16 value fits into an i17, so we can lower the width
978 // to i64.
979 .narrowScalarIf(
980 Predicate: [=](const LegalityQuery &Query) {
981 return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
982 },
983 Mutation: changeTo(TypeIdx: 0, Ty: i64))
984 .lowerIf(Predicate: ::any(P0: scalarWiderThan(TypeIdx: 0, Size: 64), P1: scalarWiderThan(TypeIdx: 1, Size: 64)), Mutation: 0)
985 .moreElementsToNextPow2(TypeIdx: 0)
986 .widenScalarToNextPow2(TypeIdx: 0, /*MinSize=*/32)
987 .minScalar(TypeIdx: 0, Ty: s32)
988 .widenScalarIf(
989 Predicate: [HasFP16](const LegalityQuery &Query) {
990 return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
991 Query.Types[1].getScalarType().isBFloat16();
992 },
993 Mutation: changeElementTo(TypeIdx: 1, Ty: f32))
994 .widenScalarIf(
995 Predicate: [=](const LegalityQuery &Query) {
996 unsigned ITySize = Query.Types[0].getScalarSizeInBits();
997 return (ITySize == 16 || ITySize == 32 || ITySize == 64) &&
998 ITySize > Query.Types[1].getScalarSizeInBits();
999 },
1000 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1001 .widenScalarIf(
1002 Predicate: [=](const LegalityQuery &Query) {
1003 unsigned FTySize = Query.Types[1].getScalarSizeInBits();
1004 return (FTySize == 16 || FTySize == 32 || FTySize == 64) &&
1005 Query.Types[0].getScalarSizeInBits() < FTySize;
1006 },
1007 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1008 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1009 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1010 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1011 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1012
1013 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
1014 .legalFor(Types: {{f32, i32},
1015 {f64, i32},
1016 {f32, i64},
1017 {f64, i64},
1018 {v2f32, v2i32},
1019 {v4f32, v4i32},
1020 {v2f64, v2i64}})
1021 .legalFor(Pred: HasFP16,
1022 Types: {{f16, i32}, {f16, i64}, {v4f16, v4i16}, {v8f16, v8i16}})
1023 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1024 return Query.Types[0].getScalarType().isBFloat16();
1025 })
1026 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1)
1027 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1028 .moreElementsToNextPow2(TypeIdx: 1)
1029 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 1)
1030 .minScalar(TypeIdx: 1, Ty: f32)
1031 .lowerIf(Predicate: [](const LegalityQuery &Query) {
1032 return Query.Types[1].isVector() &&
1033 Query.Types[1].getScalarSizeInBits() == 64 &&
1034 Query.Types[0].getScalarSizeInBits() == 16;
1035 })
1036 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, /*MinSize=*/HasFP16 ? 16 : 32)
1037 .scalarizeIf(
1038 // v2i64->v2f32 needs to scalarize to avoid double-rounding issues.
1039 Predicate: [](const LegalityQuery &Query) {
1040 return Query.Types[0].getScalarSizeInBits() == 32 &&
1041 Query.Types[1].getScalarSizeInBits() == 64;
1042 },
1043 TypeIdx: 0)
1044 .widenScalarIf(
1045 Predicate: [](const LegalityQuery &Query) {
1046 return Query.Types[1].getScalarSizeInBits() <= 64 &&
1047 Query.Types[0].getScalarSizeInBits() <
1048 Query.Types[1].getScalarSizeInBits();
1049 },
1050 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: 1))
1051 .widenScalarIf(
1052 Predicate: [](const LegalityQuery &Query) {
1053 return Query.Types[0].getScalarSizeInBits() <= 64 &&
1054 Query.Types[0].getScalarSizeInBits() >
1055 Query.Types[1].getScalarSizeInBits();
1056 },
1057 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 1, FromTypeIdx: 0))
1058 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1059 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1060 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1061 .libcallFor(Types: {{f16, i128},
1062 {f32, i128},
1063 {f64, i128},
1064 {f128, i128},
1065 {f128, i32},
1066 {f128, i64}});
1067
1068 // Control-flow
1069 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
1070 getActionDefinitionsBuilder(Opcode: G_BRCOND)
1071 .legalFor(Types: {s32})
1072 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s32);
1073 getActionDefinitionsBuilder(Opcode: G_BRINDIRECT).legalFor(Types: {p0});
1074
1075 getActionDefinitionsBuilder(Opcode: G_SELECT)
1076 .legalFor(Types: {{s32, s32}, {s64, s32}, {p0, s32}})
1077 .widenScalarToNextPow2(TypeIdx: 0)
1078 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64)
1079 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s32)
1080 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1081 .minScalarEltSameAsIf(Predicate: all(P0: isVector(TypeIdx: 0), P1: isVector(TypeIdx: 1)), TypeIdx: 1, LargeTypeIdx: 0)
1082 .lowerIf(Predicate: isVector(TypeIdx: 0));
1083
1084 // Pointer-handling
1085 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
1086
1087 if (TM.getCodeModel() == CodeModel::Small)
1088 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).custom();
1089 else
1090 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: {p0});
1091
1092 getActionDefinitionsBuilder(Opcode: G_PTRAUTH_GLOBAL_VALUE)
1093 .legalIf(Predicate: all(P0: typeIs(TypeIdx: 0, TypesInit: p0), P1: typeIs(TypeIdx: 1, TypesInit: p0)));
1094
1095 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
1096 .legalFor(Types: {{i64, p0}, {v2i64, v2p0}})
1097 .widenScalarToNextPow2(TypeIdx: 0, MinSize: 64)
1098 .clampScalar(TypeIdx: 0, MinTy: s64, MaxTy: s64)
1099 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2);
1100
1101 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
1102 .unsupportedIf(Predicate: [&](const LegalityQuery &Query) {
1103 return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
1104 })
1105 .legalFor(Types: {{p0, i64}, {v2p0, v2i64}})
1106 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2);
1107
1108 // Casts for 32 and 64-bit width type are just copies.
1109 // Same for 128-bit width type, except they are on the FPR bank.
1110 getActionDefinitionsBuilder(Opcode: G_BITCAST)
1111 .legalForCartesianProduct(Types: {s16})
1112 // Keeping 32-bit instructions legal to prevent regression in some tests
1113 .legalForCartesianProduct(Types: {s32, v2s16, v4s8})
1114 .legalForCartesianProduct(Types: {s64, v8s8, v4s16, v2s32})
1115 .legalForCartesianProduct(Types: {s128, v16s8, v8s16, v4s32, v2s64, v2p0})
1116 .customIf(Predicate: [=](const LegalityQuery &Query) {
1117 // Handle casts from i1 vectors to scalars.
1118 LLT DstTy = Query.Types[0];
1119 LLT SrcTy = Query.Types[1];
1120 return DstTy.isScalar() && SrcTy.isVector() &&
1121 SrcTy.getScalarSizeInBits() == 1;
1122 })
1123 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
1124 return Query.Types[0].isVector() != Query.Types[1].isVector();
1125 })
1126 .moreElementsToNextPow2(TypeIdx: 0)
1127 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1128 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1129 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1130 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1131 .lower();
1132
1133 getActionDefinitionsBuilder(Opcode: G_VASTART).legalFor(Types: {p0});
1134
1135 // va_list must be a pointer, but most sized types are pretty easy to handle
1136 // as the destination.
1137 getActionDefinitionsBuilder(Opcode: G_VAARG)
1138 .customForCartesianProduct(Types0: {s8, s16, s32, s64, p0}, Types1: {p0})
1139 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s64)
1140 .widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8);
1141
1142 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS)
1143 .lowerIf(
1144 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s8, s16, s32, s64, s128}), P1: typeIs(TypeIdx: 2, TypesInit: p0)));
1145
1146 bool UseOutlineAtomics = ST.outlineAtomics() && !ST.hasLSE();
1147
1148 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG)
1149 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1150 .customFor(Pred: !UseOutlineAtomics, Types: {{s128, p0}})
1151 .libcallFor(Pred: UseOutlineAtomics,
1152 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {s128, p0}})
1153 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1154
1155 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD,
1156 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
1157 G_ATOMICRMW_XOR})
1158 .legalFor(Pred: !UseOutlineAtomics, Types: {{s32, p0}, {s64, p0}})
1159 .libcallFor(Pred: UseOutlineAtomics,
1160 Types: {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
1161 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1162
1163 // Do not outline these atomics operations, as per comment in
1164 // AArch64ISelLowering.cpp's shouldExpandAtomicRMWInIR().
1165 getActionDefinitionsBuilder(
1166 Opcodes: {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
1167 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: {s32, s64}), P1: typeIs(TypeIdx: 1, TypesInit: p0)))
1168 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: s64);
1169
1170 getActionDefinitionsBuilder(Opcode: G_BLOCK_ADDR).legalFor(Types: {p0});
1171
1172 // Merge/Unmerge
1173 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
1174 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
1175 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
1176 getActionDefinitionsBuilder(Opcode: Op)
1177 .widenScalarToNextPow2(TypeIdx: LitTyIdx, MinSize: 8)
1178 .widenScalarToNextPow2(TypeIdx: BigTyIdx, MinSize: 32)
1179 .clampScalar(TypeIdx: LitTyIdx, MinTy: s8, MaxTy: s64)
1180 .clampScalar(TypeIdx: BigTyIdx, MinTy: s32, MaxTy: s128)
1181 .legalIf(Predicate: [=](const LegalityQuery &Q) {
1182 switch (Q.Types[BigTyIdx].getSizeInBits()) {
1183 case 32:
1184 case 64:
1185 case 128:
1186 break;
1187 default:
1188 return false;
1189 }
1190 switch (Q.Types[LitTyIdx].getSizeInBits()) {
1191 case 8:
1192 case 16:
1193 case 32:
1194 case 64:
1195 return true;
1196 default:
1197 return false;
1198 }
1199 });
1200 }
1201
1202 // TODO : nxv4s16, nxv2s16, nxv2s32
1203 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
1204 .legalFor(Pred: HasSVE, Types: {{s16, nxv16s8, s64},
1205 {s16, nxv8s16, s64},
1206 {s32, nxv4s32, s64},
1207 {s64, nxv2s64, s64}})
1208 .unsupportedIf(Predicate: [=](const LegalityQuery &Query) {
1209 const LLT &EltTy = Query.Types[1].getElementType();
1210 if (Query.Types[1].isScalableVector())
1211 return false;
1212 return Query.Types[0] != EltTy;
1213 })
1214 .minScalar(TypeIdx: 2, Ty: s64)
1215 .customIf(Predicate: [=](const LegalityQuery &Query) {
1216 const LLT &VecTy = Query.Types[1];
1217 return VecTy == v8s8 || VecTy == v16s8 || VecTy == v2s16 ||
1218 VecTy == v4s16 || VecTy == v8s16 || VecTy == v2s32 ||
1219 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2p0;
1220 })
1221 .minScalarOrEltIf(
1222 Predicate: [=](const LegalityQuery &Query) {
1223 // We want to promote to <M x s1> to <M x s64> if that wouldn't
1224 // cause the total vec size to be > 128b.
1225 return Query.Types[1].isFixedVector() &&
1226 Query.Types[1].getNumElements() <= 2;
1227 },
1228 TypeIdx: 0, Ty: s64)
1229 .minScalarOrEltIf(
1230 Predicate: [=](const LegalityQuery &Query) {
1231 return Query.Types[1].isFixedVector() &&
1232 Query.Types[1].getNumElements() <= 4;
1233 },
1234 TypeIdx: 0, Ty: s32)
1235 .minScalarOrEltIf(
1236 Predicate: [=](const LegalityQuery &Query) {
1237 return Query.Types[1].isFixedVector() &&
1238 Query.Types[1].getNumElements() <= 8;
1239 },
1240 TypeIdx: 0, Ty: s16)
1241 .minScalarOrEltIf(
1242 Predicate: [=](const LegalityQuery &Query) {
1243 return Query.Types[1].isFixedVector() &&
1244 Query.Types[1].getNumElements() <= 16;
1245 },
1246 TypeIdx: 0, Ty: s8)
1247 .minScalarOrElt(TypeIdx: 0, Ty: s8) // Worst case, we need at least s8.
1248 .moreElementsToNextPow2(TypeIdx: 1)
1249 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1250 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1251 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1252 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1253 .clampMaxNumElements(TypeIdx: 1, EltTy: p0, MaxElements: 2)
1254 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 1, Size: 64), TypeIdx: 1);
1255
1256 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
1257 .legalIf(
1258 Predicate: typeInSet(TypeIdx: 0, TypesInit: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64, v2p0}))
1259 .legalFor(Pred: HasSVE, Types: {{nxv16s8, s32, s64},
1260 {nxv8s16, s32, s64},
1261 {nxv4s32, s32, s64},
1262 {nxv2s64, s64, s64}})
1263 .moreElementsToNextPow2(TypeIdx: 0)
1264 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1265 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1266 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1267 .clampNumElements(TypeIdx: 0, MinTy: v2s32, MaxTy: v4s32)
1268 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 2)
1269 .clampMaxNumElements(TypeIdx: 0, EltTy: p0, MaxElements: 2)
1270 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
1271
1272 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
1273 .legalFor(Types: {{v8s8, s8},
1274 {v16s8, s8},
1275 {v4s16, s16},
1276 {v8s16, s16},
1277 {v2s32, s32},
1278 {v4s32, s32},
1279 {v2s64, s64},
1280 {v2p0, p0}})
1281 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1282 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1283 .minScalarOrElt(TypeIdx: 0, Ty: s8)
1284 .widenVectorEltsToVectorMinSize(TypeIdx: 0, VectorSize: 64)
1285 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1286 .minScalarSameAs(TypeIdx: 1, LargeTypeIdx: 0);
1287
1288 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR_TRUNC).lower();
1289
1290 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
1291 .legalIf(Predicate: [=](const LegalityQuery &Query) {
1292 const LLT &DstTy = Query.Types[0];
1293 const LLT &SrcTy = Query.Types[1];
1294 // For now just support the TBL2 variant which needs the source vectors
1295 // to be the same size as the dest.
1296 if (DstTy != SrcTy)
1297 return false;
1298 return llvm::is_contained(
1299 Set: {v8s8, v16s8, v4s16, v8s16, v2s32, v4s32, v2s64}, Element: DstTy);
1300 })
1301 .moreElementsIf(
1302 Predicate: [](const LegalityQuery &Query) {
1303 return Query.Types[0].getNumElements() >
1304 Query.Types[1].getNumElements();
1305 },
1306 Mutation: changeTo(TypeIdx: 1, FromTypeIdx: 0))
1307 .moreElementsToNextPow2(TypeIdx: 0)
1308 .moreElementsIf(
1309 Predicate: [](const LegalityQuery &Query) {
1310 return Query.Types[0].getNumElements() <
1311 Query.Types[1].getNumElements();
1312 },
1313 Mutation: changeTo(TypeIdx: 0, FromTypeIdx: 1))
1314 .widenScalarOrEltToNextPow2OrMinSize(TypeIdx: 0, MinSize: 8)
1315 .clampNumElements(TypeIdx: 0, MinTy: v8s8, MaxTy: v16s8)
1316 .clampNumElements(TypeIdx: 0, MinTy: v4s16, MaxTy: v8s16)
1317 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: v4s32)
1318 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: v2s64)
1319 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0)
1320 .bitcastIf(Predicate: isPointerVector(TypeIdx: 0), Mutation: [=](const LegalityQuery &Query) {
1321 // Bitcast pointers vector to i64.
1322 const LLT DstTy = Query.Types[0];
1323 return std::pair(
1324 0, LLT::vector(EC: DstTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: 64)));
1325 });
1326
1327 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
1328 .legalFor(Types: {{v16s8, v8s8}, {v8s16, v4s16}, {v4s32, v2s32}})
1329 .customIf(Predicate: [=](const LegalityQuery &Query) {
1330 return Query.Types[0].isFixedVector() &&
1331 Query.Types[0].getScalarSizeInBits() < 8;
1332 })
1333 .bitcastIf(
1334 Predicate: [=](const LegalityQuery &Query) {
1335 return Query.Types[0].isFixedVector() &&
1336 Query.Types[1].isFixedVector() &&
1337 Query.Types[0].getScalarSizeInBits() >= 8 &&
1338 isPowerOf2_64(Value: Query.Types[0].getScalarSizeInBits()) &&
1339 Query.Types[0].getSizeInBits() <= 128 &&
1340 Query.Types[1].getSizeInBits() <= 64;
1341 },
1342 Mutation: [=](const LegalityQuery &Query) {
1343 const LLT DstTy = Query.Types[0];
1344 const LLT SrcTy = Query.Types[1];
1345 return std::pair(
1346 0, DstTy.changeElementSize(NewEltSize: SrcTy.getSizeInBits())
1347 .changeElementCount(
1348 EC: DstTy.getElementCount().divideCoefficientBy(
1349 RHS: SrcTy.getNumElements())));
1350 });
1351
1352 getActionDefinitionsBuilder(Opcode: G_EXTRACT_SUBVECTOR)
1353 .legalFor(Types: {{v8s8, v16s8}, {v4s16, v8s16}, {v2s32, v4s32}})
1354 .widenScalarOrEltToNextPow2(TypeIdx: 0)
1355 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1356
1357 // TODO: {nxv16s8, s8}, {nxv8s16, s16}
1358 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
1359 .legalFor(Pred: HasSVE, Types: {{nxv4s32, s32}, {nxv2s64, s64}});
1360
1361 getActionDefinitionsBuilder(Opcode: G_JUMP_TABLE).legalFor(Types: {p0});
1362
1363 getActionDefinitionsBuilder(Opcode: G_BRJT).legalFor(Types: {{p0, s64}});
1364
1365 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
1366
1367 getActionDefinitionsBuilder(Opcode: G_DYN_STACKALLOC).custom();
1368
1369 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).lower();
1370
1371 if (ST.hasMOPS()) {
1372 // G_BZERO is not supported. Currently it is only emitted by
1373 // PreLegalizerCombiner for G_MEMSET with zero constant.
1374 getActionDefinitionsBuilder(Opcode: G_BZERO).unsupported();
1375
1376 getActionDefinitionsBuilder(Opcode: G_MEMSET)
1377 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1378 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64})
1379 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1380
1381 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE})
1382 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64})
1383 .immIdx(ImmIdx: 0); // Inform verifier imm idx 0 is handled.
1384
1385 // G_MEMCPY_INLINE does not have a tailcall immediate
1386 getActionDefinitionsBuilder(Opcode: G_MEMCPY_INLINE)
1387 .legalForCartesianProduct(Types0: {p0}, Types1: {p0}, Types2: {s64});
1388
1389 getActionDefinitionsBuilder(Opcode: G_MEMSET_INLINE)
1390 .legalForCartesianProduct(Types0: {p0}, Types1: {s64}, Types2: {s64})
1391 .customForCartesianProduct(Types0: {p0}, Types1: {s8}, Types2: {s64});
1392 } else {
1393 getActionDefinitionsBuilder(Opcodes: {G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
1394 .libcall();
1395 }
1396
1397 // For fadd reductions we have pairwise operations available. We treat the
1398 // usual legal types as legal and handle the lowering to pairwise instructions
1399 // later.
1400 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FADD)
1401 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1402 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1403 .widenScalarIf(
1404 Predicate: [HasFP16](const LegalityQuery &Query) {
1405 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1406 Query.Types[0].getScalarType().isBFloat16();
1407 },
1408 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1409 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1410 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1411 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1412 .moreElementsToNextPow2(TypeIdx: 1)
1413 .scalarize(TypeIdx: 1)
1414 .lower();
1415
1416 // For fmul reductions we need to split up into individual operations. We
1417 // clamp to 128 bit vectors then to 64bit vectors to produce a cascade of
1418 // smaller types, followed by scalarizing what remains.
1419 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_FMUL)
1420 .widenScalarIf(
1421 Predicate: [HasFP16](const LegalityQuery &Query) {
1422 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1423 Query.Types[0].getScalarType().isBFloat16();
1424 },
1425 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1426 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1427 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1428 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1429 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1430 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1431 .scalarize(TypeIdx: 1)
1432 .lower();
1433
1434 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
1435 .scalarize(TypeIdx: 2)
1436 .lower();
1437
1438 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_ADD)
1439 .legalFor(Types: {{i8, v8i8},
1440 {i8, v16i8},
1441 {i16, v4i16},
1442 {i16, v8i16},
1443 {i32, v2i32},
1444 {i32, v4i32},
1445 {i64, v2i64}})
1446 .moreElementsToNextPow2(TypeIdx: 1)
1447 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1448 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1449 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1450 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1451 .widenVectorEltsToVectorMinSize(TypeIdx: 1, VectorSize: 64)
1452 .scalarize(TypeIdx: 1);
1453
1454 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_FMIN, G_VECREDUCE_FMAX,
1455 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1456 .legalFor(Types: {{f32, v2f32}, {f32, v4f32}, {f64, v2f64}})
1457 .legalFor(Pred: HasFP16, Types: {{f16, v4f16}, {f16, v8f16}})
1458 .widenScalarIf(
1459 Predicate: [HasFP16](const LegalityQuery &Query) {
1460 return (!HasFP16 && Query.Types[0].getScalarType().isFloat16()) ||
1461 Query.Types[0].getScalarType().isBFloat16();
1462 },
1463 Mutation: changeElementTo(TypeIdx: 0, Ty: f32))
1464 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1465 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1466 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1467 .scalarize(TypeIdx: 1)
1468 .lower();
1469
1470 getActionDefinitionsBuilder(Opcode: G_VECREDUCE_MUL)
1471 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 2)
1472 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 4)
1473 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 8)
1474 .scalarize(TypeIdx: 1)
1475 .lower();
1476
1477 getActionDefinitionsBuilder(
1478 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1479 .legalFor(Types: {{i8, v8i8},
1480 {i8, v16i8},
1481 {i16, v4i16},
1482 {i16, v8i16},
1483 {i32, v2i32},
1484 {i32, v4i32}})
1485 .moreElementsIf(
1486 Predicate: [=](const LegalityQuery &Query) {
1487 return Query.Types[1].isVector() &&
1488 Query.Types[1].getElementType() != s8 &&
1489 Query.Types[1].getNumElements() & 1;
1490 },
1491 Mutation: LegalizeMutations::moreElementsToNextPow2(TypeIdx: 1))
1492 .clampMaxNumElements(TypeIdx: 1, EltTy: s64, MaxElements: 2)
1493 .clampMaxNumElements(TypeIdx: 1, EltTy: s32, MaxElements: 4)
1494 .clampMaxNumElements(TypeIdx: 1, EltTy: s16, MaxElements: 8)
1495 .clampMaxNumElements(TypeIdx: 1, EltTy: s8, MaxElements: 16)
1496 .scalarize(TypeIdx: 1)
1497 .lower();
1498
1499 getActionDefinitionsBuilder(
1500 Opcodes: {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1501 // Try to break down into smaller vectors as long as they're at least 64
1502 // bits. This lets us use vector operations for some parts of the
1503 // reduction.
1504 .fewerElementsIf(
1505 Predicate: [=](const LegalityQuery &Q) {
1506 LLT SrcTy = Q.Types[1];
1507 if (SrcTy.isScalar())
1508 return false;
1509 if (!isPowerOf2_32(Value: SrcTy.getNumElements()))
1510 return false;
1511 // We can usually perform 64b vector operations.
1512 return SrcTy.getSizeInBits() > 64;
1513 },
1514 Mutation: [=](const LegalityQuery &Q) {
1515 LLT SrcTy = Q.Types[1];
1516 return std::make_pair(x: 1, y: SrcTy.divide(Factor: 2));
1517 })
1518 .scalarize(TypeIdx: 1)
1519 .lower();
1520
1521 // TODO: Update this to correct handling when adding AArch64/SVE support.
1522 getActionDefinitionsBuilder(Opcode: G_VECTOR_COMPRESS).lower();
1523
1524 // Access to floating-point environment.
1525 getActionDefinitionsBuilder(Opcodes: {G_GET_FPENV, G_SET_FPENV, G_RESET_FPENV,
1526 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1527 .libcall();
1528
1529 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).lower();
1530
1531 getActionDefinitionsBuilder(Opcode: G_PREFETCH).custom();
1532
1533 getActionDefinitionsBuilder(Opcodes: {G_SCMP, G_UCMP}).lower();
1534
1535 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
1536 .alwaysLegal();
1537 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
1538 getActionDefinitionsBuilder(Opcode: G_INVOKE_REGION_START).alwaysLegal();
1539
1540 verify(MII: *ST.getInstrInfo());
1541}
1542
1543bool AArch64LegalizerInfo::legalizeCustom(
1544 LegalizerHelper &Helper, MachineInstr &MI,
1545 LostDebugLocObserver &LocObserver) const {
1546 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1547 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1548 GISelChangeObserver &Observer = Helper.Observer;
1549 switch (MI.getOpcode()) {
1550 default:
1551 // No idea what to do.
1552 return false;
1553 case TargetOpcode::G_VAARG:
1554 return legalizeVaArg(MI, MRI, MIRBuilder);
1555 case TargetOpcode::G_LOAD:
1556 case TargetOpcode::G_STORE:
1557 return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
1558 case TargetOpcode::G_SHL:
1559 case TargetOpcode::G_ASHR:
1560 case TargetOpcode::G_LSHR:
1561 return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
1562 case TargetOpcode::G_GLOBAL_VALUE:
1563 return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
1564 case TargetOpcode::G_SBFX:
1565 case TargetOpcode::G_UBFX:
1566 return legalizeBitfieldExtract(MI, MRI, Helper);
1567 case TargetOpcode::G_FSHL:
1568 case TargetOpcode::G_FSHR:
1569 return legalizeFunnelShift(MI, MRI, MIRBuilder, Observer, Helper);
1570 case TargetOpcode::G_ROTR:
1571 return legalizeRotate(MI, MRI, Helper);
1572 case TargetOpcode::G_CTPOP:
1573 return legalizeCTPOP(MI, MRI, Helper);
1574 case TargetOpcode::G_ATOMIC_CMPXCHG:
1575 return legalizeAtomicCmpxchg128(MI, MRI, Helper);
1576 case TargetOpcode::G_CTTZ:
1577 return legalizeCTTZ(MI, Helper);
1578 case TargetOpcode::G_BZERO:
1579 case TargetOpcode::G_MEMCPY:
1580 case TargetOpcode::G_MEMMOVE:
1581 case TargetOpcode::G_MEMSET:
1582 case TargetOpcode::G_MEMSET_INLINE:
1583 return legalizeMemOps(MI, Helper);
1584 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1585 return legalizeExtractVectorElt(MI, MRI, Helper);
1586 case TargetOpcode::G_DYN_STACKALLOC:
1587 return legalizeDynStackAlloc(MI, Helper);
1588 case TargetOpcode::G_PREFETCH:
1589 return legalizePrefetch(MI, Helper);
1590 case TargetOpcode::G_ABS:
1591 return Helper.lowerAbsToCNeg(MI);
1592 case TargetOpcode::G_ICMP:
1593 return legalizeICMP(MI, MRI, MIRBuilder);
1594 case TargetOpcode::G_BITCAST:
1595 return legalizeBitcast(MI, Helper);
1596 case TargetOpcode::G_CONCAT_VECTORS:
1597 return legalizeConcatVectors(MI, MRI, MIRBuilder);
1598 case TargetOpcode::G_FPTRUNC:
1599 // In order to lower f16 to f64 properly, we need to use f32 as an
1600 // intermediary
1601 return legalizeFptrunc(MI, MIRBuilder, MRI);
1602 }
1603
1604 llvm_unreachable("expected switch to return");
1605}
1606
1607bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
1608 LegalizerHelper &Helper) const {
1609 assert(MI.getOpcode() == TargetOpcode::G_BITCAST && "Unexpected opcode");
1610 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1611 // We're trying to handle casts from i1 vectors to scalars but reloading from
1612 // stack.
1613 if (!DstTy.isScalar() || !SrcTy.isVector() ||
1614 SrcTy.getElementType() != LLT::scalar(SizeInBits: 1))
1615 return false;
1616
1617 Helper.createStackStoreLoad(Res: DstReg, Val: SrcReg);
1618 MI.eraseFromParent();
1619 return true;
1620}
1621
1622bool AArch64LegalizerInfo::legalizeFunnelShift(MachineInstr &MI,
1623 MachineRegisterInfo &MRI,
1624 MachineIRBuilder &MIRBuilder,
1625 GISelChangeObserver &Observer,
1626 LegalizerHelper &Helper) const {
1627 assert(MI.getOpcode() == TargetOpcode::G_FSHL ||
1628 MI.getOpcode() == TargetOpcode::G_FSHR);
1629
1630 // Keep as G_FSHR if shift amount is a G_CONSTANT, else use generic
1631 // lowering
1632 Register ShiftNo = MI.getOperand(i: 3).getReg();
1633 LLT ShiftTy = MRI.getType(Reg: ShiftNo);
1634 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: ShiftNo, MRI);
1635
1636 // Adjust shift amount according to Opcode (FSHL/FSHR)
1637 // Convert FSHL to FSHR
1638 LLT OperationTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1639 APInt BitWidth(ShiftTy.getSizeInBits(), OperationTy.getSizeInBits(), false);
1640
1641 // Lower non-constant shifts and leave zero shifts to the optimizer.
1642 if (!VRegAndVal || VRegAndVal->Value.urem(RHS: BitWidth) == 0)
1643 return (Helper.lowerFunnelShiftAsShifts(MI) ==
1644 LegalizerHelper::LegalizeResult::Legalized);
1645
1646 APInt Amount = VRegAndVal->Value.urem(RHS: BitWidth);
1647
1648 Amount = MI.getOpcode() == TargetOpcode::G_FSHL ? BitWidth - Amount : Amount;
1649
1650 // If the instruction is G_FSHR, has a 64-bit G_CONSTANT for shift amount
1651 // in the range of 0 <-> BitWidth, it is legal
1652 if (ShiftTy.getSizeInBits() == 64 && MI.getOpcode() == TargetOpcode::G_FSHR &&
1653 VRegAndVal->Value.ult(RHS: BitWidth))
1654 return true;
1655
1656 // Cast the ShiftNumber to a 64-bit type
1657 auto Cast64 = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Amount.zext(width: 64));
1658
1659 if (MI.getOpcode() == TargetOpcode::G_FSHR) {
1660 Observer.changingInstr(MI);
1661 MI.getOperand(i: 3).setReg(Cast64.getReg(Idx: 0));
1662 Observer.changedInstr(MI);
1663 }
1664 // If Opcode is FSHL, remove the FSHL instruction and create a FSHR
1665 // instruction
1666 else if (MI.getOpcode() == TargetOpcode::G_FSHL) {
1667 MIRBuilder.buildInstr(Opc: TargetOpcode::G_FSHR, DstOps: {MI.getOperand(i: 0).getReg()},
1668 SrcOps: {MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getReg(),
1669 Cast64.getReg(Idx: 0)});
1670 MI.eraseFromParent();
1671 }
1672 return true;
1673}
1674
1675bool AArch64LegalizerInfo::legalizeICMP(MachineInstr &MI,
1676 MachineRegisterInfo &MRI,
1677 MachineIRBuilder &MIRBuilder) const {
1678 Register DstReg = MI.getOperand(i: 0).getReg();
1679 Register SrcReg1 = MI.getOperand(i: 2).getReg();
1680 Register SrcReg2 = MI.getOperand(i: 3).getReg();
1681 LLT DstTy = MRI.getType(Reg: DstReg);
1682 LLT SrcTy = MRI.getType(Reg: SrcReg1);
1683
1684 // Check the vector types are legal
1685 if (DstTy.getScalarSizeInBits() != SrcTy.getScalarSizeInBits() ||
1686 DstTy.getNumElements() != SrcTy.getNumElements() ||
1687 (DstTy.getSizeInBits() != 64 && DstTy.getSizeInBits() != 128))
1688 return false;
1689
1690 // Lowers G_ICMP NE => G_ICMP EQ to allow better pattern matching for
1691 // following passes
1692 CmpInst::Predicate Pred = (CmpInst::Predicate)MI.getOperand(i: 1).getPredicate();
1693 if (Pred != CmpInst::ICMP_NE)
1694 return true;
1695 Register CmpReg =
1696 MIRBuilder
1697 .buildICmp(Pred: CmpInst::ICMP_EQ, Res: MRI.getType(Reg: DstReg), Op0: SrcReg1, Op1: SrcReg2)
1698 .getReg(Idx: 0);
1699 MIRBuilder.buildNot(Dst: DstReg, Src0: CmpReg);
1700
1701 MI.eraseFromParent();
1702 return true;
1703}
1704
1705bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
1706 MachineRegisterInfo &MRI,
1707 LegalizerHelper &Helper) const {
1708 // To allow for imported patterns to match, we ensure that the rotate amount
1709 // is 64b with an extension.
1710 Register AmtReg = MI.getOperand(i: 2).getReg();
1711 LLT AmtTy = MRI.getType(Reg: AmtReg);
1712 (void)AmtTy;
1713 assert(AmtTy.isScalar() && "Expected a scalar rotate");
1714 assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
1715 auto NewAmt = Helper.MIRBuilder.buildZExt(Res: LLT::integer(SizeInBits: 64), Op: AmtReg);
1716 Helper.Observer.changingInstr(MI);
1717 MI.getOperand(i: 2).setReg(NewAmt.getReg(Idx: 0));
1718 Helper.Observer.changedInstr(MI);
1719 return true;
1720}
1721
1722bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1723 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
1724 GISelChangeObserver &Observer) const {
1725 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1726 // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
1727 // G_ADD_LOW instructions.
1728 // By splitting this here, we can optimize accesses in the small code model by
1729 // folding in the G_ADD_LOW into the load/store offset.
1730 auto &GlobalOp = MI.getOperand(i: 1);
1731 // Don't modify an intrinsic call.
1732 if (GlobalOp.isSymbol())
1733 return true;
1734 const auto* GV = GlobalOp.getGlobal();
1735 if (GV->isThreadLocal())
1736 return true; // Don't want to modify TLS vars.
1737
1738 auto &TM = ST->getTargetLowering()->getTargetMachine();
1739 unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
1740
1741 if (OpFlags & AArch64II::MO_GOT)
1742 return true;
1743
1744 auto Offset = GlobalOp.getOffset();
1745 Register DstReg = MI.getOperand(i: 0).getReg();
1746 auto ADRP = MIRBuilder.buildInstr(Opc: AArch64::ADRP, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {})
1747 .addGlobalAddress(GV, Offset, TargetFlags: OpFlags | AArch64II::MO_PAGE);
1748 // Set the regclass on the dest reg too.
1749 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1750
1751 // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
1752 // by creating a MOVK that sets bits 48-63 of the register to (global address
1753 // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
1754 // prevent an incorrect tag being generated during relocation when the
1755 // global appears before the code section. Without the offset, a global at
1756 // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
1757 // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
1758 // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
1759 // instead of `0xf`.
1760 // This assumes that we're in the small code model so we can assume a binary
1761 // size of <= 4GB, which makes the untagged PC relative offset positive. The
1762 // binary must also be loaded into address range [0, 2^48). Both of these
1763 // properties need to be ensured at runtime when using tagged addresses.
1764 if (OpFlags & AArch64II::MO_TAGGED) {
1765 assert(!Offset &&
1766 "Should not have folded in an offset for a tagged global!");
1767 ADRP = MIRBuilder.buildInstr(Opc: AArch64::MOVKXi, DstOps: {LLT::pointer(AddressSpace: 0, SizeInBits: 64)}, SrcOps: {ADRP})
1768 .addGlobalAddress(GV, Offset: 0x100000000,
1769 TargetFlags: AArch64II::MO_PREL | AArch64II::MO_G3)
1770 .addImm(Val: 48);
1771 MRI.setRegClass(Reg: ADRP.getReg(Idx: 0), RC: &AArch64::GPR64RegClass);
1772 }
1773
1774 MIRBuilder.buildInstr(Opc: AArch64::G_ADD_LOW, DstOps: {DstReg}, SrcOps: {ADRP})
1775 .addGlobalAddress(GV, Offset,
1776 TargetFlags: OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1777 MI.eraseFromParent();
1778 return true;
1779}
1780
1781bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1782 MachineInstr &MI) const {
1783 MachineIRBuilder &MIB = Helper.MIRBuilder;
1784 MachineRegisterInfo &MRI = *MIB.getMRI();
1785
1786 auto LowerUnaryOp = [&MI, &MIB](unsigned Opcode) {
1787 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1788 MI.eraseFromParent();
1789 return true;
1790 };
1791 auto LowerBinOp = [&MI, &MIB](unsigned Opcode) {
1792 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1793 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3)});
1794 MI.eraseFromParent();
1795 return true;
1796 };
1797 auto LowerTriOp = [&MI, &MIB](unsigned Opcode) {
1798 MIB.buildInstr(Opc: Opcode, DstOps: {MI.getOperand(i: 0)},
1799 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4)});
1800 MI.eraseFromParent();
1801 return true;
1802 };
1803
1804 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1805 switch (IntrinsicID) {
1806 case Intrinsic::vacopy: {
1807 unsigned PtrSize = ST->isTargetILP32() ? 4 : 8;
1808 unsigned VaListSize =
1809 (ST->isTargetDarwin() || ST->isTargetWindows())
1810 ? PtrSize
1811 : ST->isTargetILP32() ? 20 : 32;
1812
1813 MachineFunction &MF = *MI.getMF();
1814 auto Val = MF.getRegInfo().createGenericVirtualRegister(
1815 Ty: LLT::scalar(SizeInBits: VaListSize * 8));
1816 MIB.buildLoad(Res: Val, Addr: MI.getOperand(i: 2),
1817 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1818 F: MachineMemOperand::MOLoad,
1819 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1820 MIB.buildStore(Val, Addr: MI.getOperand(i: 1),
1821 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
1822 F: MachineMemOperand::MOStore,
1823 Size: VaListSize, BaseAlignment: Align(PtrSize)));
1824 MI.eraseFromParent();
1825 return true;
1826 }
1827 case Intrinsic::get_dynamic_area_offset: {
1828 MIB.buildConstant(Res: MI.getOperand(i: 0).getReg(), Val: 0);
1829 MI.eraseFromParent();
1830 return true;
1831 }
1832 case Intrinsic::aarch64_mops_memset_tag: {
1833 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1834 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
1835 // the instruction).
1836 auto &Value = MI.getOperand(i: 3);
1837 Register ExtValueReg = MIB.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
1838 Value.setReg(ExtValueReg);
1839 return true;
1840 }
1841 case Intrinsic::aarch64_prefetch: {
1842 auto &AddrVal = MI.getOperand(i: 1);
1843
1844 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1845 int64_t Target = MI.getOperand(i: 3).getImm();
1846 int64_t IsStream = MI.getOperand(i: 4).getImm();
1847 int64_t IsData = MI.getOperand(i: 5).getImm();
1848
1849 unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
1850 (!IsData << 3) | // IsDataCache bit
1851 (Target << 1) | // Cache level bits
1852 (unsigned)IsStream; // Stream bit
1853
1854 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
1855 MI.eraseFromParent();
1856 return true;
1857 }
1858 case Intrinsic::aarch64_range_prefetch: {
1859 auto &AddrVal = MI.getOperand(i: 1);
1860
1861 int64_t IsWrite = MI.getOperand(i: 2).getImm();
1862 int64_t IsStream = MI.getOperand(i: 3).getImm();
1863 unsigned PrfOp = (IsStream << 2) | IsWrite;
1864
1865 MIB.buildInstr(Opcode: AArch64::G_AARCH64_RANGE_PREFETCH)
1866 .addImm(Val: PrfOp)
1867 .add(MO: AddrVal)
1868 .addUse(RegNo: MI.getOperand(i: 4).getReg()); // Metadata
1869 MI.eraseFromParent();
1870 return true;
1871 }
1872 case Intrinsic::aarch64_prefetch_ir: {
1873 auto &AddrVal = MI.getOperand(i: 1);
1874 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: 24).add(MO: AddrVal);
1875 MI.eraseFromParent();
1876 return true;
1877 }
1878 case Intrinsic::aarch64_neon_uaddv:
1879 case Intrinsic::aarch64_neon_saddv:
1880 case Intrinsic::aarch64_neon_umaxv:
1881 case Intrinsic::aarch64_neon_smaxv:
1882 case Intrinsic::aarch64_neon_uminv:
1883 case Intrinsic::aarch64_neon_sminv: {
1884 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1885 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1886 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1887
1888 auto OldDst = MI.getOperand(i: 0).getReg();
1889 auto OldDstTy = MRI.getType(Reg: OldDst);
1890 LLT NewDstTy = MRI.getType(Reg: MI.getOperand(i: 2).getReg()).getElementType();
1891 if (OldDstTy == NewDstTy)
1892 return true;
1893
1894 auto NewDst = MRI.createGenericVirtualRegister(Ty: NewDstTy);
1895
1896 Helper.Observer.changingInstr(MI);
1897 MI.getOperand(i: 0).setReg(NewDst);
1898 Helper.Observer.changedInstr(MI);
1899
1900 MIB.setInsertPt(MBB&: MIB.getMBB(), II: ++MIB.getInsertPt());
1901 MIB.buildExtOrTrunc(ExtOpc: IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1902 Res: OldDst, Op: NewDst);
1903
1904 return true;
1905 }
1906 case Intrinsic::aarch64_neon_uaddlp:
1907 case Intrinsic::aarch64_neon_saddlp: {
1908 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1909 ? AArch64::G_UADDLP
1910 : AArch64::G_SADDLP;
1911 MIB.buildInstr(Opc, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
1912 MI.eraseFromParent();
1913
1914 return true;
1915 }
1916 case Intrinsic::aarch64_neon_uaddlv:
1917 case Intrinsic::aarch64_neon_saddlv: {
1918 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1919 ? AArch64::G_UADDLV
1920 : AArch64::G_SADDLV;
1921 Register DstReg = MI.getOperand(i: 0).getReg();
1922 Register SrcReg = MI.getOperand(i: 2).getReg();
1923 LLT DstTy = MRI.getType(Reg: DstReg);
1924
1925 LLT MidTy, ExtTy;
1926 if (DstTy.isScalar() && DstTy.getScalarSizeInBits() <= 32) {
1927 ExtTy = LLT::integer(SizeInBits: 32);
1928 MidTy = LLT::fixed_vector(NumElements: 4, ScalarTy: ExtTy);
1929 } else {
1930 ExtTy = LLT::integer(SizeInBits: 64);
1931 MidTy = LLT::fixed_vector(NumElements: 2, ScalarTy: ExtTy);
1932 }
1933
1934 Register MidReg =
1935 MIB.buildInstr(Opc, DstOps: {MidTy}, SrcOps: {SrcReg})->getOperand(i: 0).getReg();
1936 Register ZeroReg =
1937 MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0)->getOperand(i: 0).getReg();
1938 Register ExtReg = MIB.buildInstr(Opc: AArch64::G_EXTRACT_VECTOR_ELT, DstOps: {ExtTy},
1939 SrcOps: {MidReg, ZeroReg})
1940 .getReg(Idx: 0);
1941
1942 if (DstTy.getScalarSizeInBits() < 32)
1943 MIB.buildTrunc(Res: DstReg, Op: ExtReg);
1944 else
1945 MIB.buildCopy(Res: DstReg, Op: ExtReg);
1946
1947 MI.eraseFromParent();
1948
1949 return true;
1950 }
1951 case Intrinsic::aarch64_neon_smax:
1952 return LowerBinOp(TargetOpcode::G_SMAX);
1953 case Intrinsic::aarch64_neon_smin:
1954 return LowerBinOp(TargetOpcode::G_SMIN);
1955 case Intrinsic::aarch64_neon_umax:
1956 return LowerBinOp(TargetOpcode::G_UMAX);
1957 case Intrinsic::aarch64_neon_umin:
1958 return LowerBinOp(TargetOpcode::G_UMIN);
1959 case Intrinsic::aarch64_neon_fmax:
1960 return LowerBinOp(TargetOpcode::G_FMAXIMUM);
1961 case Intrinsic::aarch64_neon_fmin:
1962 return LowerBinOp(TargetOpcode::G_FMINIMUM);
1963 case Intrinsic::aarch64_neon_fmaxnm:
1964 return LowerBinOp(TargetOpcode::G_FMAXNUM);
1965 case Intrinsic::aarch64_neon_fminnm:
1966 return LowerBinOp(TargetOpcode::G_FMINNUM);
1967 case Intrinsic::aarch64_neon_pmul:
1968 return LowerBinOp(TargetOpcode::G_CLMUL);
1969 case Intrinsic::aarch64_neon_pmull:
1970 case Intrinsic::aarch64_neon_pmull64:
1971 return LowerBinOp(AArch64::G_PMULL);
1972 case Intrinsic::aarch64_neon_smull:
1973 return LowerBinOp(AArch64::G_SMULL);
1974 case Intrinsic::aarch64_neon_umull:
1975 return LowerBinOp(AArch64::G_UMULL);
1976 case Intrinsic::aarch64_neon_sabd:
1977 return LowerBinOp(TargetOpcode::G_ABDS);
1978 case Intrinsic::aarch64_neon_uabd:
1979 return LowerBinOp(TargetOpcode::G_ABDU);
1980 case Intrinsic::aarch64_neon_uhadd:
1981 return LowerBinOp(TargetOpcode::G_UAVGFLOOR);
1982 case Intrinsic::aarch64_neon_urhadd:
1983 return LowerBinOp(TargetOpcode::G_UAVGCEIL);
1984 case Intrinsic::aarch64_neon_shadd:
1985 return LowerBinOp(TargetOpcode::G_SAVGFLOOR);
1986 case Intrinsic::aarch64_neon_srhadd:
1987 return LowerBinOp(TargetOpcode::G_SAVGCEIL);
1988 case Intrinsic::aarch64_neon_sqshrn: {
1989 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
1990 return true;
1991 // Create right shift instruction. Store the output register in Shr.
1992 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
1993 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
1994 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
1995 // Build the narrow intrinsic, taking in Shr.
1996 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
1997 MI.eraseFromParent();
1998 return true;
1999 }
2000 case Intrinsic::aarch64_neon_sqshrun: {
2001 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2002 return true;
2003 // Create right shift instruction. Store the output register in Shr.
2004 auto Shr = MIB.buildInstr(Opc: AArch64::G_VASHR,
2005 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2006 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2007 // Build the narrow intrinsic, taking in Shr.
2008 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2009 MI.eraseFromParent();
2010 return true;
2011 }
2012 case Intrinsic::aarch64_neon_sqrshrn: {
2013 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2014 return true;
2015 // Create right shift instruction. Store the output register in Shr.
2016 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2017 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2018 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2019 // Build the narrow intrinsic, taking in Shr.
2020 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_S, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2021 MI.eraseFromParent();
2022 return true;
2023 }
2024 case Intrinsic::aarch64_neon_sqrshrun: {
2025 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2026 return true;
2027 // Create right shift instruction. Store the output register in Shr.
2028 auto Shr = MIB.buildInstr(Opc: AArch64::G_SRSHR_I,
2029 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2030 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2031 // Build the narrow intrinsic, taking in Shr.
2032 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_SSAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2033 MI.eraseFromParent();
2034 return true;
2035 }
2036 case Intrinsic::aarch64_neon_uqrshrn: {
2037 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2038 return true;
2039 // Create right shift instruction. Store the output register in Shr.
2040 auto Shr = MIB.buildInstr(Opc: AArch64::G_URSHR_I,
2041 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2042 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2043 // Build the narrow intrinsic, taking in Shr.
2044 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2045 MI.eraseFromParent();
2046 return true;
2047 }
2048 case Intrinsic::aarch64_neon_uqshrn: {
2049 if (!MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2050 return true;
2051 // Create right shift instruction. Store the output register in Shr.
2052 auto Shr = MIB.buildInstr(Opc: AArch64::G_VLSHR,
2053 DstOps: {MRI.getType(Reg: MI.getOperand(i: 2).getReg())},
2054 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3).getImm()});
2055 // Build the narrow intrinsic, taking in Shr.
2056 MIB.buildInstr(Opc: TargetOpcode::G_TRUNC_USAT_U, DstOps: {MI.getOperand(i: 0)}, SrcOps: {Shr});
2057 MI.eraseFromParent();
2058 return true;
2059 }
2060 case Intrinsic::aarch64_neon_sqshlu: {
2061 // Check if last operand is constant vector dup
2062 auto ShiftAmount =
2063 isConstantOrConstantSplatVector(Def: MI.getOperand(i: 3).getReg(), MRI);
2064 if (ShiftAmount) {
2065 // If so, create a new intrinsic with the correct shift amount
2066 MIB.buildInstr(Opc: AArch64::G_SQSHLU_I, DstOps: {MI.getOperand(i: 0)},
2067 SrcOps: {MI.getOperand(i: 2)})
2068 .addImm(Val: ShiftAmount->getSExtValue());
2069 MI.eraseFromParent();
2070 return true;
2071 }
2072 return false;
2073 }
2074 case Intrinsic::aarch64_neon_vsli: {
2075 MIB.buildInstr(
2076 Opc: AArch64::G_SLI, DstOps: {MI.getOperand(i: 0)},
2077 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2078 MI.eraseFromParent();
2079 break;
2080 }
2081 case Intrinsic::aarch64_neon_vsri: {
2082 MIB.buildInstr(
2083 Opc: AArch64::G_SRI, DstOps: {MI.getOperand(i: 0)},
2084 SrcOps: {MI.getOperand(i: 2), MI.getOperand(i: 3), MI.getOperand(i: 4).getImm()});
2085 MI.eraseFromParent();
2086 break;
2087 }
2088 case Intrinsic::aarch64_neon_abs: {
2089 // Lower the intrinsic to G_ABS.
2090 MIB.buildInstr(Opc: TargetOpcode::G_ABS, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 2)});
2091 MI.eraseFromParent();
2092 return true;
2093 }
2094 case Intrinsic::aarch64_neon_sqadd: {
2095 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2096 return LowerBinOp(TargetOpcode::G_SADDSAT);
2097 break;
2098 }
2099 case Intrinsic::aarch64_neon_sqsub: {
2100 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2101 return LowerBinOp(TargetOpcode::G_SSUBSAT);
2102 break;
2103 }
2104 case Intrinsic::aarch64_neon_uqadd: {
2105 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2106 return LowerBinOp(TargetOpcode::G_UADDSAT);
2107 break;
2108 }
2109 case Intrinsic::aarch64_neon_uqsub: {
2110 if (MRI.getType(Reg: MI.getOperand(i: 0).getReg()).isVector())
2111 return LowerBinOp(TargetOpcode::G_USUBSAT);
2112 break;
2113 }
2114 case Intrinsic::aarch64_neon_udot:
2115 return LowerTriOp(AArch64::G_UDOT);
2116 case Intrinsic::aarch64_neon_sdot:
2117 return LowerTriOp(AArch64::G_SDOT);
2118 case Intrinsic::aarch64_neon_usdot:
2119 return LowerTriOp(AArch64::G_USDOT);
2120 case Intrinsic::aarch64_neon_sqxtn:
2121 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_S);
2122 case Intrinsic::aarch64_neon_sqxtun:
2123 return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_U);
2124 case Intrinsic::aarch64_neon_uqxtn:
2125 return LowerUnaryOp(TargetOpcode::G_TRUNC_USAT_U);
2126 case Intrinsic::aarch64_neon_fcvtzu:
2127 return LowerUnaryOp(TargetOpcode::G_FPTOUI_SAT);
2128 case Intrinsic::aarch64_neon_fcvtzs:
2129 return LowerUnaryOp(TargetOpcode::G_FPTOSI_SAT);
2130 case Intrinsic::aarch64_neon_cls:
2131 return LowerUnaryOp(TargetOpcode::G_CTLS);
2132
2133 case Intrinsic::vector_reverse:
2134 // TODO: Add support for vector_reverse
2135 return false;
2136 }
2137
2138 return true;
2139}
2140
2141bool AArch64LegalizerInfo::legalizeShlAshrLshr(
2142 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2143 GISelChangeObserver &Observer) const {
2144 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
2145 MI.getOpcode() == TargetOpcode::G_LSHR ||
2146 MI.getOpcode() == TargetOpcode::G_SHL);
2147 // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
2148 // imported patterns can select it later. Either way, it will be legal.
2149 Register AmtReg = MI.getOperand(i: 2).getReg();
2150 LLT AmtRegEltTy = MRI.getType(Reg: AmtReg).getScalarType();
2151 auto VRegAndVal = getIConstantVRegValWithLookThrough(VReg: AmtReg, MRI);
2152 if (!VRegAndVal)
2153 return true;
2154 // Check the shift amount is in range for an immediate form.
2155 int64_t Amount = VRegAndVal->Value.getSExtValue();
2156 if (Amount > 31)
2157 return true; // This will have to remain a register variant.
2158 auto ExtCst =
2159 MIRBuilder.buildConstant(Res: AmtRegEltTy.changeElementSize(NewEltSize: 64), Val: Amount);
2160 Observer.changingInstr(MI);
2161 MI.getOperand(i: 2).setReg(ExtCst.getReg(Idx: 0));
2162 Observer.changedInstr(MI);
2163 return true;
2164}
2165
2166static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset,
2167 MachineRegisterInfo &MRI) {
2168 Base = Root;
2169 Offset = 0;
2170
2171 Register NewBase;
2172 int64_t NewOffset;
2173 if (mi_match(R: Root, MRI, P: m_GPtrAdd(L: m_Reg(R&: NewBase), R: m_ICst(Cst&: NewOffset))) &&
2174 isShiftedInt<7, 3>(x: NewOffset)) {
2175 Base = NewBase;
2176 Offset = NewOffset;
2177 }
2178}
2179
2180// FIXME: This should be removed and replaced with the generic bitcast legalize
2181// action.
2182bool AArch64LegalizerInfo::legalizeLoadStore(
2183 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
2184 GISelChangeObserver &Observer) const {
2185 assert(MI.getOpcode() == TargetOpcode::G_STORE ||
2186 MI.getOpcode() == TargetOpcode::G_LOAD);
2187 // Here we just try to handle vector loads/stores where our value type might
2188 // have pointer elements, which the SelectionDAG importer can't handle. To
2189 // allow the existing patterns for s64 to fire for p0, we just try to bitcast
2190 // the value to use s64 types.
2191
2192 // Custom legalization requires the instruction, if not deleted, must be fully
2193 // legalized. In order to allow further legalization of the inst, we create
2194 // a new instruction and erase the existing one.
2195
2196 Register ValReg = MI.getOperand(i: 0).getReg();
2197 const LLT ValTy = MRI.getType(Reg: ValReg);
2198
2199 if (ValTy == LLT::scalar(SizeInBits: 128)) {
2200
2201 AtomicOrdering Ordering = (*MI.memoperands_begin())->getSuccessOrdering();
2202 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2203 bool IsLoadAcquire = IsLoad && Ordering == AtomicOrdering::Acquire;
2204 bool IsStoreRelease = !IsLoad && Ordering == AtomicOrdering::Release;
2205 bool IsRcpC3 =
2206 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
2207
2208 LLT s64 = LLT::integer(SizeInBits: 64);
2209
2210 unsigned Opcode;
2211 if (IsRcpC3) {
2212 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
2213 } else {
2214 // For LSE2, loads/stores should have been converted to monotonic and had
2215 // a fence inserted after them.
2216 assert(Ordering == AtomicOrdering::Monotonic ||
2217 Ordering == AtomicOrdering::Unordered);
2218 assert(ST->hasLSE2() && "ldp/stp not single copy atomic without +lse2");
2219
2220 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
2221 }
2222
2223 MachineInstrBuilder NewI;
2224 if (IsLoad) {
2225 NewI = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {s64, s64}, SrcOps: {});
2226 MIRBuilder.buildMergeLikeInstr(
2227 Res: ValReg, Ops: {NewI->getOperand(i: 0), NewI->getOperand(i: 1)});
2228 } else {
2229 auto Split = MIRBuilder.buildUnmerge(Res: s64, Op: MI.getOperand(i: 0));
2230 NewI = MIRBuilder.buildInstr(
2231 Opc: Opcode, DstOps: {}, SrcOps: {Split->getOperand(i: 0), Split->getOperand(i: 1)});
2232 }
2233
2234 if (IsRcpC3) {
2235 NewI.addUse(RegNo: MI.getOperand(i: 1).getReg());
2236 } else {
2237 Register Base;
2238 int Offset;
2239 matchLDPSTPAddrMode(Root: MI.getOperand(i: 1).getReg(), Base, Offset, MRI);
2240 NewI.addUse(RegNo: Base);
2241 NewI.addImm(Val: Offset / 8);
2242 }
2243
2244 NewI.cloneMemRefs(OtherMI: MI);
2245 constrainSelectedInstRegOperands(I&: *NewI, TII: *ST->getInstrInfo(),
2246 TRI: *MRI.getTargetRegisterInfo(),
2247 RBI: *ST->getRegBankInfo());
2248 MI.eraseFromParent();
2249 return true;
2250 }
2251
2252 if (!ValTy.isPointerVector() ||
2253 ValTy.getElementType().getAddressSpace() != 0) {
2254 LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
2255 return false;
2256 }
2257
2258 unsigned PtrSize = ValTy.getElementType().getSizeInBits();
2259 const LLT NewTy = LLT::vector(EC: ValTy.getElementCount(), ScalarTy: LLT::integer(SizeInBits: PtrSize));
2260 auto &MMO = **MI.memoperands_begin();
2261 MMO.setType(NewTy);
2262
2263 if (MI.getOpcode() == TargetOpcode::G_STORE) {
2264 auto Bitcast = MIRBuilder.buildBitcast(Dst: NewTy, Src: ValReg);
2265 MIRBuilder.buildStore(Val: Bitcast.getReg(Idx: 0), Addr: MI.getOperand(i: 1), MMO);
2266 } else {
2267 auto NewLoad = MIRBuilder.buildLoad(Res: NewTy, Addr: MI.getOperand(i: 1), MMO);
2268 MIRBuilder.buildBitcast(Dst: ValReg, Src: NewLoad);
2269 }
2270 MI.eraseFromParent();
2271 return true;
2272}
2273
2274bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
2275 MachineRegisterInfo &MRI,
2276 MachineIRBuilder &MIRBuilder) const {
2277 MachineFunction &MF = MIRBuilder.getMF();
2278 Align Alignment(MI.getOperand(i: 2).getImm());
2279 Register Dst = MI.getOperand(i: 0).getReg();
2280 Register ListPtr = MI.getOperand(i: 1).getReg();
2281
2282 LLT PtrTy = MRI.getType(Reg: ListPtr);
2283 LLT IntPtrTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
2284
2285 const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
2286 const Align PtrAlign = Align(PtrSize);
2287 auto List = MIRBuilder.buildLoad(
2288 Res: PtrTy, Addr: ListPtr,
2289 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), f: MachineMemOperand::MOLoad,
2290 MemTy: PtrTy, base_alignment: PtrAlign));
2291
2292 MachineInstrBuilder DstPtr;
2293 if (Alignment > PtrAlign) {
2294 // Realign the list to the actual required alignment.
2295 auto AlignMinus1 =
2296 MIRBuilder.buildConstant(Res: IntPtrTy, Val: Alignment.value() - 1);
2297 auto ListTmp = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: List, Op1: AlignMinus1.getReg(Idx: 0));
2298 DstPtr = MIRBuilder.buildMaskLowPtrBits(Res: PtrTy, Op0: ListTmp, NumBits: Log2(A: Alignment));
2299 } else
2300 DstPtr = List;
2301
2302 LLT ValTy = MRI.getType(Reg: Dst);
2303 uint64_t ValSize = ValTy.getSizeInBits() / 8;
2304 MIRBuilder.buildLoad(
2305 Res: Dst, Addr: DstPtr,
2306 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(), f: MachineMemOperand::MOLoad,
2307 MemTy: ValTy, base_alignment: std::max(a: Alignment, b: PtrAlign)));
2308
2309 auto Size = MIRBuilder.buildConstant(Res: IntPtrTy, Val: alignTo(Size: ValSize, A: PtrAlign));
2310
2311 auto NewList = MIRBuilder.buildPtrAdd(Res: PtrTy, Op0: DstPtr, Op1: Size.getReg(Idx: 0));
2312
2313 MIRBuilder.buildStore(Val: NewList, Addr: ListPtr,
2314 MMO&: *MF.getMachineMemOperand(PtrInfo: MachinePointerInfo(),
2315 f: MachineMemOperand::MOStore,
2316 MemTy: PtrTy, base_alignment: PtrAlign));
2317
2318 MI.eraseFromParent();
2319 return true;
2320}
2321
2322bool AArch64LegalizerInfo::legalizeBitfieldExtract(
2323 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2324 // Only legal if we can select immediate forms.
2325 // TODO: Lower this otherwise.
2326 return getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI) &&
2327 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
2328}
2329
2330bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
2331 MachineRegisterInfo &MRI,
2332 LegalizerHelper &Helper) const {
2333 // When there is no integer popcount instruction (FEAT_CSSC isn't available),
2334 // it can be more efficiently lowered to the following sequence that uses
2335 // AdvSIMD registers/instructions as long as the copies to/from the AdvSIMD
2336 // registers are cheap.
2337 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
2338 // CNT V0.8B, V0.8B // 8xbyte pop-counts
2339 // ADDV B0, V0.8B // sum 8xbyte pop-counts
2340 // UMOV X0, V0.B[0] // copy byte result back to integer reg
2341 //
2342 // For 128 bit vector popcounts, we lower to the following sequence:
2343 // cnt.16b v0, v0 // v8s16, v4s32, v2s64
2344 // uaddlp.8h v0, v0 // v8s16, v4s32, v2s64
2345 // uaddlp.4s v0, v0 // v4s32, v2s64
2346 // uaddlp.2d v0, v0 // v2s64
2347 //
2348 // For 64 bit vector popcounts, we lower to the following sequence:
2349 // cnt.8b v0, v0 // v4s16, v2s32
2350 // uaddlp.4h v0, v0 // v4s16, v2s32
2351 // uaddlp.2s v0, v0 // v2s32
2352
2353 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2354 Register Dst = MI.getOperand(i: 0).getReg();
2355 Register Val = MI.getOperand(i: 1).getReg();
2356 LLT Ty = MRI.getType(Reg: Val);
2357
2358 LLT i64 = LLT::integer(SizeInBits: 64);
2359 LLT i32 = LLT::integer(SizeInBits: 32);
2360 LLT i16 = LLT::integer(SizeInBits: 16);
2361 LLT i8 = LLT::integer(SizeInBits: 8);
2362 unsigned Size = Ty.getSizeInBits();
2363
2364 assert(Ty == MRI.getType(Dst) &&
2365 "Expected src and dst to have the same type!");
2366
2367 if (ST->hasCSSC() && Ty.isScalar() && Size == 128) {
2368
2369 auto Split = MIRBuilder.buildUnmerge(Res: i64, Op: Val);
2370 auto CTPOP1 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 0));
2371 auto CTPOP2 = MIRBuilder.buildCTPOP(Dst: i64, Src0: Split->getOperand(i: 1));
2372 auto Add = MIRBuilder.buildAdd(Dst: i64, Src0: CTPOP1, Src1: CTPOP2);
2373
2374 MIRBuilder.buildZExt(Res: Dst, Op: Add);
2375 MI.eraseFromParent();
2376 return true;
2377 }
2378
2379 if (!ST->hasNEON() ||
2380 MI.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
2381 // Use generic lowering when custom lowering is not possible.
2382 return Ty.isScalar() && (Size == 32 || Size == 64) &&
2383 Helper.lowerBitCount(MI) ==
2384 LegalizerHelper::LegalizeResult::Legalized;
2385 }
2386
2387 // Pre-conditioning: widen Val up to the nearest vector type.
2388 // s32,s64,v4s16,v2s32 -> v8i8
2389 // v8s16,v4s32,v2s64 -> v16i8
2390 LLT VTy = Size == 128 ? LLT::fixed_vector(NumElements: 16, ScalarTy: i8) : LLT::fixed_vector(NumElements: 8, ScalarTy: i8);
2391 if (Ty.isScalar()) {
2392 assert((Size == 32 || Size == 64 || Size == 128) && "Expected only 32, 64, or 128 bit scalars!");
2393 if (Size == 32) {
2394 Val = MIRBuilder.buildZExt(Res: i64, Op: Val).getReg(Idx: 0);
2395 }
2396 }
2397 Val = MIRBuilder.buildBitcast(Dst: VTy, Src: Val).getReg(Idx: 0);
2398
2399 // Count bits in each byte-sized lane.
2400 auto CTPOP = MIRBuilder.buildCTPOP(Dst: VTy, Src0: Val);
2401
2402 // Sum across lanes.
2403 if (ST->hasDotProd() && Ty.isVector() && Ty.getNumElements() >= 2 &&
2404 Ty.getScalarSizeInBits() != 16) {
2405 LLT Dt = Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64) ? LLT::fixed_vector(NumElements: 4, ScalarTy: i32) : Ty;
2406 auto Zeros = MIRBuilder.buildConstant(Res: Dt, Val: 0);
2407 auto Ones = MIRBuilder.buildConstant(Res: VTy, Val: 1);
2408 MachineInstrBuilder Sum;
2409
2410 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2411 auto UDOT =
2412 MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2413 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UADDLP, DstOps: {Ty}, SrcOps: {UDOT});
2414 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2415 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2416 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2417 Sum = MIRBuilder.buildInstr(Opc: AArch64::G_UDOT, DstOps: {Dt}, SrcOps: {Zeros, Ones, CTPOP});
2418 } else {
2419 llvm_unreachable("unexpected vector shape");
2420 }
2421
2422 Sum->getOperand(i: 0).setReg(Dst);
2423 MI.eraseFromParent();
2424 return true;
2425 }
2426
2427 Register HSum = CTPOP.getReg(Idx: 0);
2428 unsigned Opc;
2429 SmallVector<LLT> HAddTys;
2430 if (Ty.isScalar()) {
2431 Opc = Intrinsic::aarch64_neon_uaddlv;
2432 HAddTys.push_back(Elt: i32);
2433 } else if (Ty == LLT::fixed_vector(NumElements: 8, ScalarTy: i16)) {
2434 Opc = Intrinsic::aarch64_neon_uaddlp;
2435 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2436 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i32)) {
2437 Opc = Intrinsic::aarch64_neon_uaddlp;
2438 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2439 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2440 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i64)) {
2441 Opc = Intrinsic::aarch64_neon_uaddlp;
2442 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 8, ScalarTy: i16));
2443 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i32));
2444 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i64));
2445 } else if (Ty == LLT::fixed_vector(NumElements: 4, ScalarTy: i16)) {
2446 Opc = Intrinsic::aarch64_neon_uaddlp;
2447 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2448 } else if (Ty == LLT::fixed_vector(NumElements: 2, ScalarTy: i32)) {
2449 Opc = Intrinsic::aarch64_neon_uaddlp;
2450 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 4, ScalarTy: i16));
2451 HAddTys.push_back(Elt: LLT::fixed_vector(NumElements: 2, ScalarTy: i32));
2452 } else
2453 llvm_unreachable("unexpected vector shape");
2454 MachineInstrBuilder UADD;
2455 for (LLT HTy : HAddTys) {
2456 UADD = MIRBuilder.buildIntrinsic(ID: Opc, Res: {HTy}).addUse(RegNo: HSum);
2457 HSum = UADD.getReg(Idx: 0);
2458 }
2459
2460 // Post-conditioning.
2461 if (Ty.isScalar() && (Size == 64 || Size == 128))
2462 MIRBuilder.buildZExt(Res: Dst, Op: UADD);
2463 else
2464 UADD->getOperand(i: 0).setReg(Dst);
2465 MI.eraseFromParent();
2466 return true;
2467}
2468
2469bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
2470 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2471 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2472 LLT i64 = LLT::integer(SizeInBits: 64);
2473 auto Addr = MI.getOperand(i: 1).getReg();
2474 auto DesiredI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 2));
2475 auto NewI = MIRBuilder.buildUnmerge(Res: {i64, i64}, Op: MI.getOperand(i: 3));
2476 auto DstLo = MRI.createGenericVirtualRegister(Ty: i64);
2477 auto DstHi = MRI.createGenericVirtualRegister(Ty: i64);
2478
2479 MachineInstrBuilder CAS;
2480 if (ST->hasLSE()) {
2481 // We have 128-bit CASP instructions taking XSeqPair registers, which are
2482 // s128. We need the merge/unmerge to bracket the expansion and pair up with
2483 // the rest of the MIR so we must reassemble the extracted registers into a
2484 // 128-bit known-regclass one with code like this:
2485 //
2486 // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
2487 // %out = CASP %in1, ...
2488 // %OldLo = G_EXTRACT %out, 0
2489 // %OldHi = G_EXTRACT %out, 64
2490 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2491 unsigned Opcode;
2492 switch (Ordering) {
2493 case AtomicOrdering::Acquire:
2494 Opcode = AArch64::CASPAX;
2495 break;
2496 case AtomicOrdering::Release:
2497 Opcode = AArch64::CASPLX;
2498 break;
2499 case AtomicOrdering::AcquireRelease:
2500 case AtomicOrdering::SequentiallyConsistent:
2501 Opcode = AArch64::CASPALX;
2502 break;
2503 default:
2504 Opcode = AArch64::CASPX;
2505 break;
2506 }
2507
2508 LLT s128 = LLT::scalar(SizeInBits: 128);
2509 auto CASDst = MRI.createGenericVirtualRegister(Ty: s128);
2510 auto CASDesired = MRI.createGenericVirtualRegister(Ty: s128);
2511 auto CASNew = MRI.createGenericVirtualRegister(Ty: s128);
2512 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASDesired}, SrcOps: {})
2513 .addUse(RegNo: DesiredI->getOperand(i: 0).getReg())
2514 .addImm(Val: AArch64::sube64)
2515 .addUse(RegNo: DesiredI->getOperand(i: 1).getReg())
2516 .addImm(Val: AArch64::subo64);
2517 MIRBuilder.buildInstr(Opc: TargetOpcode::REG_SEQUENCE, DstOps: {CASNew}, SrcOps: {})
2518 .addUse(RegNo: NewI->getOperand(i: 0).getReg())
2519 .addImm(Val: AArch64::sube64)
2520 .addUse(RegNo: NewI->getOperand(i: 1).getReg())
2521 .addImm(Val: AArch64::subo64);
2522
2523 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {CASDst}, SrcOps: {CASDesired, CASNew, Addr});
2524
2525 MIRBuilder.buildExtract(Res: {DstLo}, Src: {CASDst}, Index: 0);
2526 MIRBuilder.buildExtract(Res: {DstHi}, Src: {CASDst}, Index: 64);
2527 } else {
2528 // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
2529 // can take arbitrary registers so it just has the normal GPR64 operands the
2530 // rest of AArch64 is expecting.
2531 auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
2532 unsigned Opcode;
2533 switch (Ordering) {
2534 case AtomicOrdering::Acquire:
2535 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
2536 break;
2537 case AtomicOrdering::Release:
2538 Opcode = AArch64::CMP_SWAP_128_RELEASE;
2539 break;
2540 case AtomicOrdering::AcquireRelease:
2541 case AtomicOrdering::SequentiallyConsistent:
2542 Opcode = AArch64::CMP_SWAP_128;
2543 break;
2544 default:
2545 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
2546 break;
2547 }
2548
2549 auto Scratch = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
2550 CAS = MIRBuilder.buildInstr(Opc: Opcode, DstOps: {DstLo, DstHi, Scratch},
2551 SrcOps: {Addr, DesiredI->getOperand(i: 0),
2552 DesiredI->getOperand(i: 1), NewI->getOperand(i: 0),
2553 NewI->getOperand(i: 1)});
2554 }
2555
2556 CAS.cloneMemRefs(OtherMI: MI);
2557 constrainSelectedInstRegOperands(I&: *CAS, TII: *ST->getInstrInfo(),
2558 TRI: *MRI.getTargetRegisterInfo(),
2559 RBI: *ST->getRegBankInfo());
2560
2561 MIRBuilder.buildMergeLikeInstr(Res: MI.getOperand(i: 0), Ops: {DstLo, DstHi});
2562 MI.eraseFromParent();
2563 return true;
2564}
2565
2566bool AArch64LegalizerInfo::legalizeCTTZ(MachineInstr &MI,
2567 LegalizerHelper &Helper) const {
2568 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2569 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2570 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
2571 auto BitReverse = MIRBuilder.buildBitReverse(Dst: Ty, Src: MI.getOperand(i: 1));
2572 MIRBuilder.buildCTLZ(Dst: MI.getOperand(i: 0).getReg(), Src0: BitReverse);
2573 MI.eraseFromParent();
2574 return true;
2575}
2576
2577bool AArch64LegalizerInfo::legalizeMemOps(MachineInstr &MI,
2578 LegalizerHelper &Helper) const {
2579 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2580
2581 // Tagged version MOPSMemorySetTagged is legalised in legalizeIntrinsic
2582 if (MI.getOpcode() == TargetOpcode::G_MEMSET ||
2583 MI.getOpcode() == TargetOpcode::G_MEMSET_INLINE) {
2584 // Anyext the value being set to 64 bit (only the bottom 8 bits are read by
2585 // the instruction).
2586 auto &Value = MI.getOperand(i: 1);
2587 Register ExtValueReg =
2588 MIRBuilder.buildAnyExt(Res: LLT::integer(SizeInBits: 64), Op: Value).getReg(Idx: 0);
2589 Value.setReg(ExtValueReg);
2590 return true;
2591 }
2592
2593 return false;
2594}
2595
2596bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2597 MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const {
2598 const GExtractVectorElement *Element = cast<GExtractVectorElement>(Val: &MI);
2599 auto VRegAndVal =
2600 getIConstantVRegValWithLookThrough(VReg: Element->getIndexReg(), MRI);
2601 if (VRegAndVal)
2602 return true;
2603 LLT VecTy = MRI.getType(Reg: Element->getVectorReg());
2604 if (VecTy.isScalableVector())
2605 return true;
2606 return Helper.lowerExtractInsertVectorElt(MI) !=
2607 LegalizerHelper::LegalizeResult::UnableToLegalize;
2608}
2609
2610bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2611 MachineInstr &MI, LegalizerHelper &Helper) const {
2612 MachineFunction &MF = *MI.getParent()->getParent();
2613 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
2614 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
2615
2616 // If stack probing is not enabled for this function, use the default
2617 // lowering.
2618 if (!MF.getFunction().hasFnAttribute(Kind: "probe-stack") ||
2619 MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() !=
2620 "inline-asm") {
2621 Helper.lowerDynStackAlloc(MI);
2622 return true;
2623 }
2624
2625 Register Dst = MI.getOperand(i: 0).getReg();
2626 Register AllocSize = MI.getOperand(i: 1).getReg();
2627 Align Alignment = assumeAligned(Value: MI.getOperand(i: 2).getImm());
2628
2629 assert(MRI.getType(Dst) == LLT::pointer(0, 64) &&
2630 "Unexpected type for dynamic alloca");
2631 assert(MRI.getType(AllocSize) == LLT::scalar(64) &&
2632 "Unexpected type for dynamic alloca");
2633
2634 LLT PtrTy = MRI.getType(Reg: Dst);
2635 Register SPReg =
2636 Helper.getTargetLowering().getStackPointerRegisterToSaveRestore();
2637 Register SPTmp =
2638 Helper.getDynStackAllocTargetPtr(SPReg, AllocSize, Alignment, PtrTy);
2639 auto NewMI =
2640 MIRBuilder.buildInstr(Opc: AArch64::PROBED_STACKALLOC_DYN, DstOps: {}, SrcOps: {SPTmp});
2641 MRI.setRegClass(Reg: NewMI.getReg(Idx: 0), RC: &AArch64::GPR64commonRegClass);
2642 MIRBuilder.setInsertPt(MBB&: *NewMI->getParent(), II: NewMI);
2643 MIRBuilder.buildCopy(Res: Dst, Op: SPTmp);
2644
2645 MI.eraseFromParent();
2646 return true;
2647}
2648
2649bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI,
2650 LegalizerHelper &Helper) const {
2651 MachineIRBuilder &MIB = Helper.MIRBuilder;
2652 auto &AddrVal = MI.getOperand(i: 0);
2653
2654 int64_t IsWrite = MI.getOperand(i: 1).getImm();
2655 int64_t Locality = MI.getOperand(i: 2).getImm();
2656 int64_t IsData = MI.getOperand(i: 3).getImm();
2657
2658 bool IsStream = Locality == 0;
2659 if (Locality != 0) {
2660 assert(Locality <= 3 && "Prefetch locality out-of-range");
2661 // The locality degree is the opposite of the cache speed.
2662 // Put the number the other way around.
2663 // The encoding starts at 0 for level 1
2664 Locality = 3 - Locality;
2665 }
2666
2667 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2668
2669 MIB.buildInstr(Opcode: AArch64::G_AARCH64_PREFETCH).addImm(Val: PrfOp).add(MO: AddrVal);
2670 MI.eraseFromParent();
2671 return true;
2672}
2673
2674bool AArch64LegalizerInfo::legalizeConcatVectors(
2675 MachineInstr &MI, MachineRegisterInfo &MRI,
2676 MachineIRBuilder &MIRBuilder) const {
2677 // Widen sub-byte element vectors to byte-sized elements before concatenating.
2678 // This is analogous to SDAG's integer type promotion for sub-byte types.
2679 auto &Concat = cast<GConcatVectors>(Val&: MI);
2680 Register DstReg = Concat.getReg(Idx: 0);
2681 LLT DstTy = MRI.getType(Reg: DstReg);
2682 assert(DstTy.getScalarSizeInBits() < 8 && "Expected dst ty to be < 8b");
2683
2684 unsigned WideEltSize =
2685 std::max(a: 8u, b: (unsigned)PowerOf2Ceil(A: DstTy.getScalarSizeInBits()));
2686 LLT SrcTy = MRI.getType(Reg: Concat.getSourceReg(I: 0));
2687 LLT WideSrcTy = SrcTy.changeElementSize(NewEltSize: WideEltSize);
2688 LLT WideDstTy = DstTy.changeElementSize(NewEltSize: WideEltSize);
2689
2690 SmallVector<Register> WideSrcs;
2691 for (unsigned I = 0; I < Concat.getNumSources(); ++I) {
2692 auto Wide = MIRBuilder.buildAnyExt(Res: WideSrcTy, Op: Concat.getSourceReg(I));
2693 WideSrcs.push_back(Elt: Wide.getReg(Idx: 0));
2694 }
2695
2696 auto WideConcat = MIRBuilder.buildConcatVectors(Res: WideDstTy, Ops: WideSrcs);
2697 MIRBuilder.buildTrunc(Res: DstReg, Op: WideConcat);
2698 MI.eraseFromParent();
2699 return true;
2700}
2701
2702bool AArch64LegalizerInfo::legalizeFptrunc(MachineInstr &MI,
2703 MachineIRBuilder &MIRBuilder,
2704 MachineRegisterInfo &MRI) const {
2705 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
2706
2707 // This function legalizes f64 -> bf16 and f64 -> f16 truncations via f64 ->
2708 // f32 G_FPTRUNC_ODD and f32 -> [b]f16 G_FPTRUNC, which apparently avoids the
2709 // usual double-rounding issue that could be present from using twin
2710 // G_FPTRUNC.
2711
2712 if (DstTy.isBFloat16() && SrcTy.isFloat64()) {
2713 auto Mid = MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {LLT::float32()},
2714 SrcOps: {Src}, Flags: MI.getFlags());
2715 MIRBuilder.buildInstr(Opc: AArch64::G_FPTRUNC, DstOps: {Dst}, SrcOps: {Mid}, Flags: MI.getFlags());
2716 MI.eraseFromParent();
2717 return true;
2718 }
2719
2720 assert(SrcTy.isFixedVector() && isPowerOf2_32(SrcTy.getNumElements()) &&
2721 "Expected a power of 2 elements");
2722
2723 // We must mutate types here as FPTrunc may be used on a IEEE floating point
2724 // or a brainfloat.
2725 LLT v2s16 = DstTy.changeElementCount(NumElements: 2);
2726 LLT v4s16 = DstTy.changeElementCount(NumElements: 4);
2727 LLT v2s32 = SrcTy.changeElementCount(NumElements: 2).changeElementSize(NewEltSize: 32);
2728 LLT v4s32 = SrcTy.changeElementCount(NumElements: 4).changeElementSize(NewEltSize: 32);
2729 LLT v2s64 = SrcTy.changeElementCount(NumElements: 2);
2730
2731 SmallVector<Register> RegsToUnmergeTo;
2732 SmallVector<Register> TruncOddDstRegs;
2733 SmallVector<Register> RegsToMerge;
2734
2735 unsigned ElemCount = SrcTy.getNumElements();
2736
2737 // Find the biggest size chunks we can work with
2738 int StepSize = ElemCount % 4 ? 2 : 4;
2739
2740 // If we have a power of 2 greater than 2, we need to first unmerge into
2741 // enough pieces
2742 if (ElemCount <= 2)
2743 RegsToUnmergeTo.push_back(Elt: Src);
2744 else {
2745 for (unsigned i = 0; i < ElemCount / 2; ++i)
2746 RegsToUnmergeTo.push_back(Elt: MRI.createGenericVirtualRegister(Ty: v2s64));
2747
2748 MIRBuilder.buildUnmerge(Res: RegsToUnmergeTo, Op: Src);
2749 }
2750
2751 // Create all of the round-to-odd instructions and store them
2752 for (auto SrcReg : RegsToUnmergeTo) {
2753 Register Mid = MIRBuilder
2754 .buildInstr(Opc: AArch64::G_FPTRUNC_ODD, DstOps: {v2s32}, SrcOps: {SrcReg},
2755 Flags: MI.getFlags())
2756 .getReg(Idx: 0);
2757 TruncOddDstRegs.push_back(Elt: Mid);
2758 }
2759
2760 // Truncate 4s32 to 4s16 if we can to reduce instruction count, otherwise
2761 // truncate 2s32 to 2s16.
2762 unsigned Index = 0;
2763 for (unsigned LoopIter = 0; LoopIter < ElemCount / StepSize; ++LoopIter) {
2764 if (StepSize == 4) {
2765 Register ConcatDst =
2766 MIRBuilder
2767 .buildMergeLikeInstr(
2768 Res: {v4s32}, Ops: {TruncOddDstRegs[Index++], TruncOddDstRegs[Index++]})
2769 .getReg(Idx: 0);
2770
2771 RegsToMerge.push_back(
2772 Elt: MIRBuilder.buildFPTrunc(Res: v4s16, Op: ConcatDst, Flags: MI.getFlags()).getReg(Idx: 0));
2773 } else {
2774 RegsToMerge.push_back(
2775 Elt: MIRBuilder
2776 .buildFPTrunc(Res: v2s16, Op: TruncOddDstRegs[Index++], Flags: MI.getFlags())
2777 .getReg(Idx: 0));
2778 }
2779 }
2780
2781 // If there is only one register, replace the destination
2782 if (RegsToMerge.size() == 1) {
2783 MRI.replaceRegWith(FromReg: Dst, ToReg: RegsToMerge.pop_back_val());
2784 MI.eraseFromParent();
2785 return true;
2786 }
2787
2788 // Merge the rest of the instructions & replace the register
2789 Register Fin = MIRBuilder.buildMergeLikeInstr(Res: DstTy, Ops: RegsToMerge).getReg(Idx: 0);
2790 MRI.replaceRegWith(FromReg: Dst, ToReg: Fin);
2791 MI.eraseFromParent();
2792 return true;
2793}
2794