1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_simm8_32b(int64_t Imm) {
57 return Imm >= -128 && Imm < 128;
58}
59static bool Predicate_simm8_64b(int64_t Imm) {
60 return Imm >= -128 && Imm < 128;
61}
62static bool Predicate_uimm8_32b(int64_t Imm) {
63 return Imm >= 0 && Imm < 256;
64}
65static bool Predicate_uimm8_64b(int64_t Imm) {
66 return Imm >= 0 && Imm < 256;
67}
68static bool Predicate_simm6_32b(int64_t Imm) {
69 return Imm >= -32 && Imm < 32;
70}
71
72
73// FastEmit functions for AArch64ISD::ENTRY_PSTATE_SM.
74
75Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(MVT RetVT) {
76 if (RetVT.SimpleTy != MVT::i64)
77 return Register();
78 return fastEmitInst_(MachineInstOpcode: AArch64::EntryPStateSM, RC: &AArch64::GPR64RegClass);
79}
80
81Register fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(MVT VT, MVT RetVT) {
82 switch (VT.SimpleTy) {
83 case MVT::i64: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_MVT_i64_(RetVT);
84 default: return Register();
85 }
86}
87
88// FastEmit functions for AArch64ISD::THREAD_POINTER.
89
90Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
91 if (RetVT.SimpleTy != MVT::i64)
92 return Register();
93 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
94}
95
96Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
97 switch (VT.SimpleTy) {
98 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
99 default: return Register();
100 }
101}
102
103// Top-level FastEmit function.
104
105Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
106 switch (Opcode) {
107 case AArch64ISD::ENTRY_PSTATE_SM: return fastEmit_AArch64ISD_ENTRY_PSTATE_SM_(VT, RetVT);
108 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
109 default: return Register();
110 }
111}
112
113// FastEmit functions for AArch64ISD::CALL.
114
115Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
116 if (RetVT.SimpleTy != MVT::isVoid)
117 return Register();
118 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
119 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
120 }
121 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
122 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
123 }
124 return Register();
125}
126
127Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
128 switch (VT.SimpleTy) {
129 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
130 default: return Register();
131 }
132}
133
134// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
135
136Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
137 if (RetVT.SimpleTy != MVT::bf16)
138 return Register();
139 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
140}
141
142Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
143 if (RetVT.SimpleTy != MVT::f16)
144 return Register();
145 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
146}
147
148Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
149 if (RetVT.SimpleTy != MVT::f32)
150 return Register();
151 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
152}
153
154Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
155 if (RetVT.SimpleTy != MVT::f64)
156 return Register();
157 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
158}
159
160Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
161 if (RetVT.SimpleTy != MVT::f128)
162 return Register();
163 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
164}
165
166Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
167 if (RetVT.SimpleTy != MVT::v8i8)
168 return Register();
169 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
170}
171
172Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
173 if (RetVT.SimpleTy != MVT::v16i8)
174 return Register();
175 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
176}
177
178Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
179 if (RetVT.SimpleTy != MVT::v4i16)
180 return Register();
181 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
182}
183
184Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
185 if (RetVT.SimpleTy != MVT::v8i16)
186 return Register();
187 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
188}
189
190Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
191 if (RetVT.SimpleTy != MVT::v2i32)
192 return Register();
193 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
194}
195
196Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
197 if (RetVT.SimpleTy != MVT::v4i32)
198 return Register();
199 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
200}
201
202Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
203 if (RetVT.SimpleTy != MVT::v1i64)
204 return Register();
205 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
206}
207
208Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
209 if (RetVT.SimpleTy != MVT::v2i64)
210 return Register();
211 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
212}
213
214Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
215 if (RetVT.SimpleTy != MVT::v4f16)
216 return Register();
217 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
218}
219
220Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
221 if (RetVT.SimpleTy != MVT::v8f16)
222 return Register();
223 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
224}
225
226Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
227 if (RetVT.SimpleTy != MVT::v4bf16)
228 return Register();
229 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
230}
231
232Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
233 if (RetVT.SimpleTy != MVT::v8bf16)
234 return Register();
235 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
236}
237
238Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
239 if (RetVT.SimpleTy != MVT::v2f32)
240 return Register();
241 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
242}
243
244Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
245 if (RetVT.SimpleTy != MVT::v4f32)
246 return Register();
247 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
248}
249
250Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
251 if (RetVT.SimpleTy != MVT::v1f64)
252 return Register();
253 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
254}
255
256Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
257 if (RetVT.SimpleTy != MVT::v2f64)
258 return Register();
259 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
260}
261
262Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
263 switch (VT.SimpleTy) {
264 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
265 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
266 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
267 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
268 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
269 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
270 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
271 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
272 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
273 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
274 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
275 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
276 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
277 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
278 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
279 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
280 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
281 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
282 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
283 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
284 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
285 default: return Register();
286 }
287}
288
289// FastEmit functions for AArch64ISD::DUP.
290
291Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
292 if ((Subtarget->isNeonAvailable())) {
293 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
294 }
295 return Register();
296}
297
298Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
299 if ((Subtarget->isNeonAvailable())) {
300 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
301 }
302 return Register();
303}
304
305Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
306 if ((Subtarget->isNeonAvailable())) {
307 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
308 }
309 return Register();
310}
311
312Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
313 if ((Subtarget->isNeonAvailable())) {
314 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
315 }
316 return Register();
317}
318
319Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
320 if ((Subtarget->isNeonAvailable())) {
321 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
322 }
323 return Register();
324}
325
326Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
327 if ((Subtarget->isNeonAvailable())) {
328 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
329 }
330 return Register();
331}
332
333Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
334switch (RetVT.SimpleTy) {
335 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
336 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
337 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
338 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
339 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
340 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
341 default: return Register();
342}
343}
344
345Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
346 if (RetVT.SimpleTy != MVT::v2i64)
347 return Register();
348 if ((Subtarget->isNeonAvailable())) {
349 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
350 }
351 return Register();
352}
353
354Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
355 switch (VT.SimpleTy) {
356 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
357 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
358 default: return Register();
359 }
360}
361
362// FastEmit functions for AArch64ISD::FCVTXN.
363
364Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
365 if (RetVT.SimpleTy != MVT::f32)
366 return Register();
367 if ((Subtarget->isNeonAvailable())) {
368 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
369 }
370 return Register();
371}
372
373Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
374 if (RetVT.SimpleTy != MVT::v2f32)
375 return Register();
376 if ((Subtarget->isNeonAvailable())) {
377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
378 }
379 return Register();
380}
381
382Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
383 switch (VT.SimpleTy) {
384 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
385 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
386 default: return Register();
387 }
388}
389
390// FastEmit functions for AArch64ISD::FRECPE.
391
392Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
393 if (RetVT.SimpleTy != MVT::v2f32)
394 return Register();
395 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
396}
397
398Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
399 if (RetVT.SimpleTy != MVT::v4f32)
400 return Register();
401 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
402}
403
404Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
405 if (RetVT.SimpleTy != MVT::v2f64)
406 return Register();
407 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
408}
409
410Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
411 if (RetVT.SimpleTy != MVT::nxv8f16)
412 return Register();
413 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
414 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
415 }
416 return Register();
417}
418
419Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
420 if (RetVT.SimpleTy != MVT::nxv4f32)
421 return Register();
422 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
423 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
424 }
425 return Register();
426}
427
428Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
429 if (RetVT.SimpleTy != MVT::nxv2f64)
430 return Register();
431 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
432 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
433 }
434 return Register();
435}
436
437Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
438 switch (VT.SimpleTy) {
439 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
440 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
441 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
442 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
443 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
444 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
445 default: return Register();
446 }
447}
448
449// FastEmit functions for AArch64ISD::FRSQRTE.
450
451Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
452 if (RetVT.SimpleTy != MVT::v2f32)
453 return Register();
454 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
455}
456
457Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
458 if (RetVT.SimpleTy != MVT::v4f32)
459 return Register();
460 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
461}
462
463Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
464 if (RetVT.SimpleTy != MVT::v2f64)
465 return Register();
466 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
467}
468
469Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
470 if (RetVT.SimpleTy != MVT::nxv8f16)
471 return Register();
472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
473 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
474 }
475 return Register();
476}
477
478Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
479 if (RetVT.SimpleTy != MVT::nxv4f32)
480 return Register();
481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
482 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
483 }
484 return Register();
485}
486
487Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
488 if (RetVT.SimpleTy != MVT::nxv2f64)
489 return Register();
490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
491 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
492 }
493 return Register();
494}
495
496Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
497 switch (VT.SimpleTy) {
498 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
499 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
500 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
501 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
502 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
503 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
504 default: return Register();
505 }
506}
507
508// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
509
510Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
511 if (RetVT.SimpleTy != MVT::isVoid)
512 return Register();
513 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
514}
515
516Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
517 switch (VT.SimpleTy) {
518 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
519 default: return Register();
520 }
521}
522
523// FastEmit functions for AArch64ISD::REV16.
524
525Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
526 if (RetVT.SimpleTy != MVT::i32)
527 return Register();
528 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
529}
530
531Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
532 if (RetVT.SimpleTy != MVT::i64)
533 return Register();
534 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
535}
536
537Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
538 switch (VT.SimpleTy) {
539 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
540 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
541 default: return Register();
542 }
543}
544
545// FastEmit functions for AArch64ISD::REV32.
546
547Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
548 if (RetVT.SimpleTy != MVT::v8i8)
549 return Register();
550 if ((Subtarget->isNeonAvailable())) {
551 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
552 }
553 return Register();
554}
555
556Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
557 if (RetVT.SimpleTy != MVT::v16i8)
558 return Register();
559 if ((Subtarget->isNeonAvailable())) {
560 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
561 }
562 return Register();
563}
564
565Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
566 if (RetVT.SimpleTy != MVT::v4i16)
567 return Register();
568 if ((Subtarget->isNeonAvailable())) {
569 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
570 }
571 return Register();
572}
573
574Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
575 if (RetVT.SimpleTy != MVT::v8i16)
576 return Register();
577 if ((Subtarget->isNeonAvailable())) {
578 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
579 }
580 return Register();
581}
582
583Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
584 if (RetVT.SimpleTy != MVT::v4f16)
585 return Register();
586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
587}
588
589Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
590 if (RetVT.SimpleTy != MVT::v8f16)
591 return Register();
592 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
593}
594
595Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
596 if (RetVT.SimpleTy != MVT::v4bf16)
597 return Register();
598 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
599}
600
601Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
602 if (RetVT.SimpleTy != MVT::v8bf16)
603 return Register();
604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
605}
606
607Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
608 switch (VT.SimpleTy) {
609 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
610 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
611 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
612 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
613 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
614 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
615 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
616 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
617 default: return Register();
618 }
619}
620
621// FastEmit functions for AArch64ISD::REV64.
622
623Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
624 if (RetVT.SimpleTy != MVT::v8i8)
625 return Register();
626 if ((Subtarget->isNeonAvailable())) {
627 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
628 }
629 return Register();
630}
631
632Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
633 if (RetVT.SimpleTy != MVT::v16i8)
634 return Register();
635 if ((Subtarget->isNeonAvailable())) {
636 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
637 }
638 return Register();
639}
640
641Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
642 if (RetVT.SimpleTy != MVT::v4i16)
643 return Register();
644 if ((Subtarget->isNeonAvailable())) {
645 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
646 }
647 return Register();
648}
649
650Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
651 if (RetVT.SimpleTy != MVT::v8i16)
652 return Register();
653 if ((Subtarget->isNeonAvailable())) {
654 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
655 }
656 return Register();
657}
658
659Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
660 if (RetVT.SimpleTy != MVT::v2i32)
661 return Register();
662 if ((Subtarget->isNeonAvailable())) {
663 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
664 }
665 return Register();
666}
667
668Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
669 if (RetVT.SimpleTy != MVT::v4i32)
670 return Register();
671 if ((Subtarget->isNeonAvailable())) {
672 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
673 }
674 return Register();
675}
676
677Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
678 if (RetVT.SimpleTy != MVT::v4f16)
679 return Register();
680 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
681}
682
683Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
684 if (RetVT.SimpleTy != MVT::v8f16)
685 return Register();
686 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
687}
688
689Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
690 if (RetVT.SimpleTy != MVT::v4bf16)
691 return Register();
692 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
693}
694
695Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
696 if (RetVT.SimpleTy != MVT::v8bf16)
697 return Register();
698 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
699}
700
701Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
702 if (RetVT.SimpleTy != MVT::v2f32)
703 return Register();
704 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
705}
706
707Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
708 if (RetVT.SimpleTy != MVT::v4f32)
709 return Register();
710 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
711}
712
713Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
714 switch (VT.SimpleTy) {
715 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
716 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
717 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
718 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
719 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
720 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
721 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
722 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
723 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
724 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
725 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
726 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
727 default: return Register();
728 }
729}
730
731// FastEmit functions for AArch64ISD::SADDLP.
732
733Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
734 if (RetVT.SimpleTy != MVT::v4i16)
735 return Register();
736 if ((Subtarget->isNeonAvailable())) {
737 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
738 }
739 return Register();
740}
741
742Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
743 if (RetVT.SimpleTy != MVT::v8i16)
744 return Register();
745 if ((Subtarget->isNeonAvailable())) {
746 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
747 }
748 return Register();
749}
750
751Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
752 if (RetVT.SimpleTy != MVT::v2i32)
753 return Register();
754 if ((Subtarget->isNeonAvailable())) {
755 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
756 }
757 return Register();
758}
759
760Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
761 if (RetVT.SimpleTy != MVT::v4i32)
762 return Register();
763 if ((Subtarget->isNeonAvailable())) {
764 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
765 }
766 return Register();
767}
768
769Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
770 if (RetVT.SimpleTy != MVT::v1i64)
771 return Register();
772 if ((Subtarget->isNeonAvailable())) {
773 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
774 }
775 return Register();
776}
777
778Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
779 if (RetVT.SimpleTy != MVT::v2i64)
780 return Register();
781 if ((Subtarget->isNeonAvailable())) {
782 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
783 }
784 return Register();
785}
786
787Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
788 switch (VT.SimpleTy) {
789 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
790 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
791 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
792 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
793 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
794 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
795 default: return Register();
796 }
797}
798
799// FastEmit functions for AArch64ISD::SITOF.
800
801Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
802 if (RetVT.SimpleTy != MVT::f16)
803 return Register();
804 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
805 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
806 }
807 return Register();
808}
809
810Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
811 if (RetVT.SimpleTy != MVT::f32)
812 return Register();
813 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
814 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
815 }
816 return Register();
817}
818
819Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
820 if (RetVT.SimpleTy != MVT::f64)
821 return Register();
822 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
823 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
824 }
825 return Register();
826}
827
828Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
829 switch (VT.SimpleTy) {
830 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
831 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
832 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
833 default: return Register();
834 }
835}
836
837// FastEmit functions for AArch64ISD::SQABS.
838
839Register fastEmit_AArch64ISD_SQABS_MVT_f32_r(MVT RetVT, Register Op0) {
840 if (RetVT.SimpleTy != MVT::f32)
841 return Register();
842 if ((Subtarget->isNeonAvailable())) {
843 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i32, RC: &AArch64::FPR32RegClass, Op0);
844 }
845 return Register();
846}
847
848Register fastEmit_AArch64ISD_SQABS_MVT_f64_r(MVT RetVT, Register Op0) {
849 if (RetVT.SimpleTy != MVT::f64)
850 return Register();
851 if ((Subtarget->isNeonAvailable())) {
852 return fastEmitInst_r(MachineInstOpcode: AArch64::SQABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
853 }
854 return Register();
855}
856
857Register fastEmit_AArch64ISD_SQABS_r(MVT VT, MVT RetVT, Register Op0) {
858 switch (VT.SimpleTy) {
859 case MVT::f32: return fastEmit_AArch64ISD_SQABS_MVT_f32_r(RetVT, Op0);
860 case MVT::f64: return fastEmit_AArch64ISD_SQABS_MVT_f64_r(RetVT, Op0);
861 default: return Register();
862 }
863}
864
865// FastEmit functions for AArch64ISD::SQNEG.
866
867Register fastEmit_AArch64ISD_SQNEG_MVT_f32_r(MVT RetVT, Register Op0) {
868 if (RetVT.SimpleTy != MVT::f32)
869 return Register();
870 if ((Subtarget->isNeonAvailable())) {
871 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i32, RC: &AArch64::FPR32RegClass, Op0);
872 }
873 return Register();
874}
875
876Register fastEmit_AArch64ISD_SQNEG_MVT_f64_r(MVT RetVT, Register Op0) {
877 if (RetVT.SimpleTy != MVT::f64)
878 return Register();
879 if ((Subtarget->isNeonAvailable())) {
880 return fastEmitInst_r(MachineInstOpcode: AArch64::SQNEGv1i64, RC: &AArch64::FPR64RegClass, Op0);
881 }
882 return Register();
883}
884
885Register fastEmit_AArch64ISD_SQNEG_r(MVT VT, MVT RetVT, Register Op0) {
886 switch (VT.SimpleTy) {
887 case MVT::f32: return fastEmit_AArch64ISD_SQNEG_MVT_f32_r(RetVT, Op0);
888 case MVT::f64: return fastEmit_AArch64ISD_SQNEG_MVT_f64_r(RetVT, Op0);
889 default: return Register();
890 }
891}
892
893// FastEmit functions for AArch64ISD::SUNPKHI.
894
895Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
896 if (RetVT.SimpleTy != MVT::nxv8i16)
897 return Register();
898 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
899 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
900 }
901 return Register();
902}
903
904Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
905 if (RetVT.SimpleTy != MVT::nxv4i32)
906 return Register();
907 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
908 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
909 }
910 return Register();
911}
912
913Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
914 if (RetVT.SimpleTy != MVT::nxv2i64)
915 return Register();
916 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
917 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
918 }
919 return Register();
920}
921
922Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
923 switch (VT.SimpleTy) {
924 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
925 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
926 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
927 default: return Register();
928 }
929}
930
931// FastEmit functions for AArch64ISD::SUNPKLO.
932
933Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
934 if (RetVT.SimpleTy != MVT::nxv8i16)
935 return Register();
936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
937 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
938 }
939 return Register();
940}
941
942Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
943 if (RetVT.SimpleTy != MVT::nxv4i32)
944 return Register();
945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
946 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
947 }
948 return Register();
949}
950
951Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
952 if (RetVT.SimpleTy != MVT::nxv2i64)
953 return Register();
954 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
955 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
956 }
957 return Register();
958}
959
960Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
961 switch (VT.SimpleTy) {
962 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
963 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
964 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
965 default: return Register();
966 }
967}
968
969// FastEmit functions for AArch64ISD::UADDLP.
970
971Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
972 if (RetVT.SimpleTy != MVT::v4i16)
973 return Register();
974 if ((Subtarget->isNeonAvailable())) {
975 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
976 }
977 return Register();
978}
979
980Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
981 if (RetVT.SimpleTy != MVT::v8i16)
982 return Register();
983 if ((Subtarget->isNeonAvailable())) {
984 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
985 }
986 return Register();
987}
988
989Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
990 if (RetVT.SimpleTy != MVT::v2i32)
991 return Register();
992 if ((Subtarget->isNeonAvailable())) {
993 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
994 }
995 return Register();
996}
997
998Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
999 if (RetVT.SimpleTy != MVT::v4i32)
1000 return Register();
1001 if ((Subtarget->isNeonAvailable())) {
1002 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1003 }
1004 return Register();
1005}
1006
1007Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1008 if (RetVT.SimpleTy != MVT::v1i64)
1009 return Register();
1010 if ((Subtarget->isNeonAvailable())) {
1011 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1012 }
1013 return Register();
1014}
1015
1016Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1017 if (RetVT.SimpleTy != MVT::v2i64)
1018 return Register();
1019 if ((Subtarget->isNeonAvailable())) {
1020 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1021 }
1022 return Register();
1023}
1024
1025Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1026 switch (VT.SimpleTy) {
1027 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1028 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1029 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1030 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1031 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1032 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1033 default: return Register();
1034 }
1035}
1036
1037// FastEmit functions for AArch64ISD::UITOF.
1038
1039Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1040 if (RetVT.SimpleTy != MVT::f16)
1041 return Register();
1042 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1043 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1044 }
1045 return Register();
1046}
1047
1048Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1049 if (RetVT.SimpleTy != MVT::f32)
1050 return Register();
1051 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1052 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1053 }
1054 return Register();
1055}
1056
1057Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1058 if (RetVT.SimpleTy != MVT::f64)
1059 return Register();
1060 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
1061 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1062 }
1063 return Register();
1064}
1065
1066Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1067 switch (VT.SimpleTy) {
1068 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1069 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1070 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1071 default: return Register();
1072 }
1073}
1074
1075// FastEmit functions for AArch64ISD::UUNPKHI.
1076
1077Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1078 if (RetVT.SimpleTy != MVT::nxv8i16)
1079 return Register();
1080 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1081 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1082 }
1083 return Register();
1084}
1085
1086Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1087 if (RetVT.SimpleTy != MVT::nxv4i32)
1088 return Register();
1089 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1090 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1091 }
1092 return Register();
1093}
1094
1095Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1096 if (RetVT.SimpleTy != MVT::nxv2i64)
1097 return Register();
1098 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1099 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1100 }
1101 return Register();
1102}
1103
1104Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1105 switch (VT.SimpleTy) {
1106 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1107 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1108 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1109 default: return Register();
1110 }
1111}
1112
1113// FastEmit functions for AArch64ISD::UUNPKLO.
1114
1115Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1116 if (RetVT.SimpleTy != MVT::nxv8i16)
1117 return Register();
1118 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1119 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1120 }
1121 return Register();
1122}
1123
1124Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1125 if (RetVT.SimpleTy != MVT::nxv4i32)
1126 return Register();
1127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1128 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1129 }
1130 return Register();
1131}
1132
1133Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1134 if (RetVT.SimpleTy != MVT::nxv2i64)
1135 return Register();
1136 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1137 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1138 }
1139 return Register();
1140}
1141
1142Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1143 switch (VT.SimpleTy) {
1144 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1145 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1146 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1147 default: return Register();
1148 }
1149}
1150
1151// FastEmit functions for ISD::ABS.
1152
1153Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1154 if (RetVT.SimpleTy != MVT::i32)
1155 return Register();
1156 if ((Subtarget->hasCSSC())) {
1157 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1158 }
1159 return Register();
1160}
1161
1162Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1163 if (RetVT.SimpleTy != MVT::i64)
1164 return Register();
1165 if ((!Subtarget->hasCSSC())) {
1166 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1167 }
1168 if ((Subtarget->hasCSSC())) {
1169 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1170 }
1171 return Register();
1172}
1173
1174Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1175 if (RetVT.SimpleTy != MVT::v8i8)
1176 return Register();
1177 if ((Subtarget->isNeonAvailable())) {
1178 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1179 }
1180 return Register();
1181}
1182
1183Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1184 if (RetVT.SimpleTy != MVT::v16i8)
1185 return Register();
1186 if ((Subtarget->isNeonAvailable())) {
1187 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1188 }
1189 return Register();
1190}
1191
1192Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1193 if (RetVT.SimpleTy != MVT::v4i16)
1194 return Register();
1195 if ((Subtarget->isNeonAvailable())) {
1196 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1197 }
1198 return Register();
1199}
1200
1201Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1202 if (RetVT.SimpleTy != MVT::v8i16)
1203 return Register();
1204 if ((Subtarget->isNeonAvailable())) {
1205 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1206 }
1207 return Register();
1208}
1209
1210Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1211 if (RetVT.SimpleTy != MVT::v2i32)
1212 return Register();
1213 if ((Subtarget->isNeonAvailable())) {
1214 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1215 }
1216 return Register();
1217}
1218
1219Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1220 if (RetVT.SimpleTy != MVT::v4i32)
1221 return Register();
1222 if ((Subtarget->isNeonAvailable())) {
1223 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1224 }
1225 return Register();
1226}
1227
1228Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1229 if (RetVT.SimpleTy != MVT::v1i64)
1230 return Register();
1231 if ((Subtarget->isNeonAvailable())) {
1232 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1233 }
1234 return Register();
1235}
1236
1237Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1238 if (RetVT.SimpleTy != MVT::v2i64)
1239 return Register();
1240 if ((Subtarget->isNeonAvailable())) {
1241 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1242 }
1243 return Register();
1244}
1245
1246Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1247 switch (VT.SimpleTy) {
1248 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1249 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1250 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1251 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1252 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1253 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1254 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1255 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1256 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1257 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1258 default: return Register();
1259 }
1260}
1261
1262// FastEmit functions for ISD::BITCAST.
1263
1264Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1265 if ((!Subtarget->isLittleEndian())) {
1266 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1267 }
1268 return Register();
1269}
1270
1271Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1272 if ((!Subtarget->isLittleEndian())) {
1273 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1274 }
1275 return Register();
1276}
1277
1278Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1279 if ((!Subtarget->isLittleEndian())) {
1280 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1281 }
1282 return Register();
1283}
1284
1285Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1286 if ((!Subtarget->isLittleEndian())) {
1287 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1288 }
1289 return Register();
1290}
1291
1292Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1293 if ((!Subtarget->isLittleEndian())) {
1294 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1295 }
1296 return Register();
1297}
1298
1299Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1300 if ((!Subtarget->isLittleEndian())) {
1301 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1302 }
1303 return Register();
1304}
1305
1306Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1307switch (RetVT.SimpleTy) {
1308 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1309 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1310 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1311 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1312 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1313 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1314 default: return Register();
1315}
1316}
1317
1318Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1319 if ((!Subtarget->isLittleEndian())) {
1320 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1321 }
1322 return Register();
1323}
1324
1325Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1326 if ((!Subtarget->isLittleEndian())) {
1327 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1328 }
1329 return Register();
1330}
1331
1332Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1333 if ((!Subtarget->isLittleEndian())) {
1334 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1335 }
1336 return Register();
1337}
1338
1339Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1340 if ((!Subtarget->isLittleEndian())) {
1341 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1342 }
1343 return Register();
1344}
1345
1346Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1347 if ((!Subtarget->isLittleEndian())) {
1348 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1349 }
1350 return Register();
1351}
1352
1353Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1354 if ((!Subtarget->isLittleEndian())) {
1355 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1356 }
1357 return Register();
1358}
1359
1360Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1361 if ((!Subtarget->isLittleEndian())) {
1362 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1363 }
1364 return Register();
1365}
1366
1367Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1368 if ((!Subtarget->isLittleEndian())) {
1369 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1370 }
1371 return Register();
1372}
1373
1374Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1375switch (RetVT.SimpleTy) {
1376 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1377 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1378 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1379 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1380 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1381 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1382 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1383 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1384 default: return Register();
1385}
1386}
1387
1388Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1389 if ((!Subtarget->isLittleEndian())) {
1390 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1391 }
1392 return Register();
1393}
1394
1395Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1396 if ((!Subtarget->isLittleEndian())) {
1397 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1398 }
1399 return Register();
1400}
1401
1402Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1403 if ((!Subtarget->isLittleEndian())) {
1404 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1405 }
1406 return Register();
1407}
1408
1409Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1410 if ((!Subtarget->isLittleEndian())) {
1411 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1412 }
1413 return Register();
1414}
1415
1416Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1417 if ((!Subtarget->isLittleEndian())) {
1418 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1419 }
1420 return Register();
1421}
1422
1423Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1424 if ((!Subtarget->isLittleEndian())) {
1425 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1431 if ((!Subtarget->isLittleEndian())) {
1432 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1433 }
1434 return Register();
1435}
1436
1437Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1438switch (RetVT.SimpleTy) {
1439 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1440 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1441 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1442 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1443 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1444 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1445 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1446 default: return Register();
1447}
1448}
1449
1450Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1451 if ((!Subtarget->isLittleEndian())) {
1452 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1453 }
1454 return Register();
1455}
1456
1457Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1458 if ((!Subtarget->isLittleEndian())) {
1459 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1460 }
1461 return Register();
1462}
1463
1464Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1465 if ((!Subtarget->isLittleEndian())) {
1466 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1467 }
1468 return Register();
1469}
1470
1471Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1472 if ((!Subtarget->isLittleEndian())) {
1473 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1474 }
1475 return Register();
1476}
1477
1478Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1479 if ((!Subtarget->isLittleEndian())) {
1480 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1481 }
1482 return Register();
1483}
1484
1485Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1486 if ((!Subtarget->isLittleEndian())) {
1487 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1488 }
1489 return Register();
1490}
1491
1492Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1493switch (RetVT.SimpleTy) {
1494 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1495 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1496 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1497 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1498 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1499 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1500 default: return Register();
1501}
1502}
1503
1504Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1505 if ((!Subtarget->isLittleEndian())) {
1506 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1507 }
1508 return Register();
1509}
1510
1511Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1512 if ((!Subtarget->isLittleEndian())) {
1513 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1514 }
1515 return Register();
1516}
1517
1518Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1519 if ((!Subtarget->isLittleEndian())) {
1520 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1521 }
1522 return Register();
1523}
1524
1525Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1526 if ((!Subtarget->isLittleEndian())) {
1527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1528 }
1529 return Register();
1530}
1531
1532Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1533 if ((!Subtarget->isLittleEndian())) {
1534 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1535 }
1536 return Register();
1537}
1538
1539Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1540switch (RetVT.SimpleTy) {
1541 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1542 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1543 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1544 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1545 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1546 default: return Register();
1547}
1548}
1549
1550Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1551 if ((!Subtarget->isLittleEndian())) {
1552 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1553 }
1554 return Register();
1555}
1556
1557Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1558 if ((!Subtarget->isLittleEndian())) {
1559 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1560 }
1561 return Register();
1562}
1563
1564Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1565 if ((!Subtarget->isLittleEndian())) {
1566 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1567 }
1568 return Register();
1569}
1570
1571Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1572 if ((!Subtarget->isLittleEndian())) {
1573 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1574 }
1575 return Register();
1576}
1577
1578Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1579 if ((!Subtarget->isLittleEndian())) {
1580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1581 }
1582 return Register();
1583}
1584
1585Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1586 if ((!Subtarget->isLittleEndian())) {
1587 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1588 }
1589 return Register();
1590}
1591
1592Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1593 if ((!Subtarget->isLittleEndian())) {
1594 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1595 }
1596 return Register();
1597}
1598
1599Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1600switch (RetVT.SimpleTy) {
1601 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1602 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1603 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1604 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1605 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1606 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1607 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1608 default: return Register();
1609}
1610}
1611
1612Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1613 if ((!Subtarget->isLittleEndian())) {
1614 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1615 }
1616 return Register();
1617}
1618
1619Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1620 if ((!Subtarget->isLittleEndian())) {
1621 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1622 }
1623 return Register();
1624}
1625
1626Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1627 if ((!Subtarget->isLittleEndian())) {
1628 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1629 }
1630 return Register();
1631}
1632
1633Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1634 if ((!Subtarget->isLittleEndian())) {
1635 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1636 }
1637 return Register();
1638}
1639
1640Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1641 if ((!Subtarget->isLittleEndian())) {
1642 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1643 }
1644 return Register();
1645}
1646
1647Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1648 if ((!Subtarget->isLittleEndian())) {
1649 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1650 }
1651 return Register();
1652}
1653
1654Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1655switch (RetVT.SimpleTy) {
1656 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1657 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1658 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1659 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1660 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1661 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1662 default: return Register();
1663}
1664}
1665
1666Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1667 if ((!Subtarget->isLittleEndian())) {
1668 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1669 }
1670 return Register();
1671}
1672
1673Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1674 if ((!Subtarget->isLittleEndian())) {
1675 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1676 }
1677 return Register();
1678}
1679
1680Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1681 if ((!Subtarget->isLittleEndian())) {
1682 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1683 }
1684 return Register();
1685}
1686
1687Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1688 if ((!Subtarget->isLittleEndian())) {
1689 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1690 }
1691 return Register();
1692}
1693
1694Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1695 if ((!Subtarget->isLittleEndian())) {
1696 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1697 }
1698 return Register();
1699}
1700
1701Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1702 if ((!Subtarget->isLittleEndian())) {
1703 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1704 }
1705 return Register();
1706}
1707
1708Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1709switch (RetVT.SimpleTy) {
1710 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1711 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1712 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1713 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1714 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1715 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1716 default: return Register();
1717}
1718}
1719
1720Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1721 if ((!Subtarget->isLittleEndian())) {
1722 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1723 }
1724 return Register();
1725}
1726
1727Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1728 if ((!Subtarget->isLittleEndian())) {
1729 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1730 }
1731 return Register();
1732}
1733
1734Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1735 if ((!Subtarget->isLittleEndian())) {
1736 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1737 }
1738 return Register();
1739}
1740
1741Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1742 if ((!Subtarget->isLittleEndian())) {
1743 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1744 }
1745 return Register();
1746}
1747
1748Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1749 if ((!Subtarget->isLittleEndian())) {
1750 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1751 }
1752 return Register();
1753}
1754
1755Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1756 if ((!Subtarget->isLittleEndian())) {
1757 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1758 }
1759 return Register();
1760}
1761
1762Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1763switch (RetVT.SimpleTy) {
1764 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1765 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1766 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1767 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1768 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1769 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1770 default: return Register();
1771}
1772}
1773
1774Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1775 if ((!Subtarget->isLittleEndian())) {
1776 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1777 }
1778 return Register();
1779}
1780
1781Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1782 if ((!Subtarget->isLittleEndian())) {
1783 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1784 }
1785 return Register();
1786}
1787
1788Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1789 if ((!Subtarget->isLittleEndian())) {
1790 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1791 }
1792 return Register();
1793}
1794
1795Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1796 if ((!Subtarget->isLittleEndian())) {
1797 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1798 }
1799 return Register();
1800}
1801
1802Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1803 if ((!Subtarget->isLittleEndian())) {
1804 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1805 }
1806 return Register();
1807}
1808
1809Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1810 if ((!Subtarget->isLittleEndian())) {
1811 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1812 }
1813 return Register();
1814}
1815
1816Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1817switch (RetVT.SimpleTy) {
1818 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1819 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1820 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1821 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1822 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1823 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1824 default: return Register();
1825}
1826}
1827
1828Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1829 if ((!Subtarget->isLittleEndian())) {
1830 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1831 }
1832 return Register();
1833}
1834
1835Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1836 if ((!Subtarget->isLittleEndian())) {
1837 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1838 }
1839 return Register();
1840}
1841
1842Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1843 if ((!Subtarget->isLittleEndian())) {
1844 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1845 }
1846 return Register();
1847}
1848
1849Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1850 if ((!Subtarget->isLittleEndian())) {
1851 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1852 }
1853 return Register();
1854}
1855
1856Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1857 if ((!Subtarget->isLittleEndian())) {
1858 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1859 }
1860 return Register();
1861}
1862
1863Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1864switch (RetVT.SimpleTy) {
1865 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1866 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1867 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1868 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1869 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1870 default: return Register();
1871}
1872}
1873
1874Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1875 if ((!Subtarget->isLittleEndian())) {
1876 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1877 }
1878 return Register();
1879}
1880
1881Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1882 if ((!Subtarget->isLittleEndian())) {
1883 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1884 }
1885 return Register();
1886}
1887
1888Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1889 if ((!Subtarget->isLittleEndian())) {
1890 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1891 }
1892 return Register();
1893}
1894
1895Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1896 if ((!Subtarget->isLittleEndian())) {
1897 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1898 }
1899 return Register();
1900}
1901
1902Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1903 if ((!Subtarget->isLittleEndian())) {
1904 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1905 }
1906 return Register();
1907}
1908
1909Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
1910 if ((!Subtarget->isLittleEndian())) {
1911 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1912 }
1913 return Register();
1914}
1915
1916Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
1917switch (RetVT.SimpleTy) {
1918 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
1919 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
1920 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
1921 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
1922 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
1923 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
1924 default: return Register();
1925}
1926}
1927
1928Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
1929 if ((!Subtarget->isLittleEndian())) {
1930 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1931 }
1932 return Register();
1933}
1934
1935Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
1936 if ((!Subtarget->isLittleEndian())) {
1937 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1938 }
1939 return Register();
1940}
1941
1942Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
1943 if ((!Subtarget->isLittleEndian())) {
1944 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1945 }
1946 return Register();
1947}
1948
1949Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
1950 if ((!Subtarget->isLittleEndian())) {
1951 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1952 }
1953 return Register();
1954}
1955
1956Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
1957 if ((!Subtarget->isLittleEndian())) {
1958 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1959 }
1960 return Register();
1961}
1962
1963Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
1964switch (RetVT.SimpleTy) {
1965 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
1966 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
1967 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
1968 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
1969 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
1970 default: return Register();
1971}
1972}
1973
1974Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
1975 if ((!Subtarget->isLittleEndian())) {
1976 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1977 }
1978 return Register();
1979}
1980
1981Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
1982 if ((!Subtarget->isLittleEndian())) {
1983 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1984 }
1985 return Register();
1986}
1987
1988Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
1989 if ((!Subtarget->isLittleEndian())) {
1990 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1991 }
1992 return Register();
1993}
1994
1995Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
1996 if ((!Subtarget->isLittleEndian())) {
1997 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1998 }
1999 return Register();
2000}
2001
2002Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2003 if ((!Subtarget->isLittleEndian())) {
2004 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2005 }
2006 return Register();
2007}
2008
2009Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2010 if ((!Subtarget->isLittleEndian())) {
2011 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2012 }
2013 return Register();
2014}
2015
2016Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2017 if ((!Subtarget->isLittleEndian())) {
2018 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2019 }
2020 return Register();
2021}
2022
2023Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2024switch (RetVT.SimpleTy) {
2025 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2026 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2027 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2028 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2029 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2030 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2031 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2032 default: return Register();
2033}
2034}
2035
2036Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2037 if ((!Subtarget->isLittleEndian())) {
2038 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2039 }
2040 return Register();
2041}
2042
2043Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2044 if ((!Subtarget->isLittleEndian())) {
2045 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2046 }
2047 return Register();
2048}
2049
2050Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2051 if ((!Subtarget->isLittleEndian())) {
2052 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2053 }
2054 return Register();
2055}
2056
2057Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2058 if ((!Subtarget->isLittleEndian())) {
2059 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2060 }
2061 return Register();
2062}
2063
2064Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2065 if ((!Subtarget->isLittleEndian())) {
2066 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2067 }
2068 return Register();
2069}
2070
2071Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2072 if ((!Subtarget->isLittleEndian())) {
2073 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2074 }
2075 return Register();
2076}
2077
2078Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2079switch (RetVT.SimpleTy) {
2080 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2081 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2082 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2083 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2084 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2085 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2086 default: return Register();
2087}
2088}
2089
2090Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2091 if ((!Subtarget->isLittleEndian())) {
2092 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2093 }
2094 return Register();
2095}
2096
2097Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2098 if ((!Subtarget->isLittleEndian())) {
2099 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2100 }
2101 return Register();
2102}
2103
2104Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2105 if ((!Subtarget->isLittleEndian())) {
2106 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2107 }
2108 return Register();
2109}
2110
2111Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2112 if ((!Subtarget->isLittleEndian())) {
2113 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2114 }
2115 return Register();
2116}
2117
2118Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2119 if ((!Subtarget->isLittleEndian())) {
2120 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2121 }
2122 return Register();
2123}
2124
2125Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2126 if ((!Subtarget->isLittleEndian())) {
2127 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2128 }
2129 return Register();
2130}
2131
2132Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2133switch (RetVT.SimpleTy) {
2134 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2135 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2136 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2137 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2138 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2139 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2140 default: return Register();
2141}
2142}
2143
2144Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2145 if ((!Subtarget->isLittleEndian())) {
2146 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2147 }
2148 return Register();
2149}
2150
2151Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2152 if ((!Subtarget->isLittleEndian())) {
2153 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2154 }
2155 return Register();
2156}
2157
2158Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2159 if ((!Subtarget->isLittleEndian())) {
2160 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2161 }
2162 return Register();
2163}
2164
2165Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2166 if ((!Subtarget->isLittleEndian())) {
2167 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2168 }
2169 return Register();
2170}
2171
2172Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2173 if ((!Subtarget->isLittleEndian())) {
2174 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2175 }
2176 return Register();
2177}
2178
2179Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2180 if ((!Subtarget->isLittleEndian())) {
2181 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2182 }
2183 return Register();
2184}
2185
2186Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2187switch (RetVT.SimpleTy) {
2188 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2189 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2190 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2191 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2192 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2193 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2194 default: return Register();
2195}
2196}
2197
2198Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2199 switch (VT.SimpleTy) {
2200 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2201 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2202 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2203 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2204 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2205 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2206 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2207 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2208 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2209 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2210 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2211 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2212 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2213 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2214 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2215 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2216 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2217 default: return Register();
2218 }
2219}
2220
2221// FastEmit functions for ISD::BITREVERSE.
2222
2223Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2224 if (RetVT.SimpleTy != MVT::i32)
2225 return Register();
2226 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2227}
2228
2229Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2230 if (RetVT.SimpleTy != MVT::i64)
2231 return Register();
2232 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2233}
2234
2235Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2236 if (RetVT.SimpleTy != MVT::v8i8)
2237 return Register();
2238 if ((Subtarget->isNeonAvailable())) {
2239 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2240 }
2241 return Register();
2242}
2243
2244Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2245 if (RetVT.SimpleTy != MVT::v16i8)
2246 return Register();
2247 if ((Subtarget->isNeonAvailable())) {
2248 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2249 }
2250 return Register();
2251}
2252
2253Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2254 switch (VT.SimpleTy) {
2255 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2256 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2257 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2258 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2259 default: return Register();
2260 }
2261}
2262
2263// FastEmit functions for ISD::BRIND.
2264
2265Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2266 if (RetVT.SimpleTy != MVT::isVoid)
2267 return Register();
2268 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2269}
2270
2271Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2272 switch (VT.SimpleTy) {
2273 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2274 default: return Register();
2275 }
2276}
2277
2278// FastEmit functions for ISD::BSWAP.
2279
2280Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2281 if (RetVT.SimpleTy != MVT::i32)
2282 return Register();
2283 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2284}
2285
2286Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2287 if (RetVT.SimpleTy != MVT::i64)
2288 return Register();
2289 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2290}
2291
2292Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2293 if (RetVT.SimpleTy != MVT::v4i16)
2294 return Register();
2295 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2296}
2297
2298Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2299 if (RetVT.SimpleTy != MVT::v8i16)
2300 return Register();
2301 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2302}
2303
2304Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2305 if (RetVT.SimpleTy != MVT::v2i32)
2306 return Register();
2307 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2308}
2309
2310Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2311 if (RetVT.SimpleTy != MVT::v4i32)
2312 return Register();
2313 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2314}
2315
2316Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2317 if (RetVT.SimpleTy != MVT::v2i64)
2318 return Register();
2319 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2320}
2321
2322Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2323 switch (VT.SimpleTy) {
2324 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2325 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2326 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2327 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2328 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2329 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2330 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2331 default: return Register();
2332 }
2333}
2334
2335// FastEmit functions for ISD::CTLS.
2336
2337Register fastEmit_ISD_CTLS_MVT_i32_r(MVT RetVT, Register Op0) {
2338 if (RetVT.SimpleTy != MVT::i32)
2339 return Register();
2340 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSWr, RC: &AArch64::GPR32RegClass, Op0);
2341}
2342
2343Register fastEmit_ISD_CTLS_MVT_i64_r(MVT RetVT, Register Op0) {
2344 if (RetVT.SimpleTy != MVT::i64)
2345 return Register();
2346 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSXr, RC: &AArch64::GPR64RegClass, Op0);
2347}
2348
2349Register fastEmit_ISD_CTLS_MVT_v8i8_r(MVT RetVT, Register Op0) {
2350 if (RetVT.SimpleTy != MVT::v8i8)
2351 return Register();
2352 if ((Subtarget->isNeonAvailable())) {
2353 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2354 }
2355 return Register();
2356}
2357
2358Register fastEmit_ISD_CTLS_MVT_v16i8_r(MVT RetVT, Register Op0) {
2359 if (RetVT.SimpleTy != MVT::v16i8)
2360 return Register();
2361 if ((Subtarget->isNeonAvailable())) {
2362 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2363 }
2364 return Register();
2365}
2366
2367Register fastEmit_ISD_CTLS_MVT_v4i16_r(MVT RetVT, Register Op0) {
2368 if (RetVT.SimpleTy != MVT::v4i16)
2369 return Register();
2370 if ((Subtarget->isNeonAvailable())) {
2371 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2372 }
2373 return Register();
2374}
2375
2376Register fastEmit_ISD_CTLS_MVT_v8i16_r(MVT RetVT, Register Op0) {
2377 if (RetVT.SimpleTy != MVT::v8i16)
2378 return Register();
2379 if ((Subtarget->isNeonAvailable())) {
2380 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2381 }
2382 return Register();
2383}
2384
2385Register fastEmit_ISD_CTLS_MVT_v2i32_r(MVT RetVT, Register Op0) {
2386 if (RetVT.SimpleTy != MVT::v2i32)
2387 return Register();
2388 if ((Subtarget->isNeonAvailable())) {
2389 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2390 }
2391 return Register();
2392}
2393
2394Register fastEmit_ISD_CTLS_MVT_v4i32_r(MVT RetVT, Register Op0) {
2395 if (RetVT.SimpleTy != MVT::v4i32)
2396 return Register();
2397 if ((Subtarget->isNeonAvailable())) {
2398 return fastEmitInst_r(MachineInstOpcode: AArch64::CLSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2399 }
2400 return Register();
2401}
2402
2403Register fastEmit_ISD_CTLS_r(MVT VT, MVT RetVT, Register Op0) {
2404 switch (VT.SimpleTy) {
2405 case MVT::i32: return fastEmit_ISD_CTLS_MVT_i32_r(RetVT, Op0);
2406 case MVT::i64: return fastEmit_ISD_CTLS_MVT_i64_r(RetVT, Op0);
2407 case MVT::v8i8: return fastEmit_ISD_CTLS_MVT_v8i8_r(RetVT, Op0);
2408 case MVT::v16i8: return fastEmit_ISD_CTLS_MVT_v16i8_r(RetVT, Op0);
2409 case MVT::v4i16: return fastEmit_ISD_CTLS_MVT_v4i16_r(RetVT, Op0);
2410 case MVT::v8i16: return fastEmit_ISD_CTLS_MVT_v8i16_r(RetVT, Op0);
2411 case MVT::v2i32: return fastEmit_ISD_CTLS_MVT_v2i32_r(RetVT, Op0);
2412 case MVT::v4i32: return fastEmit_ISD_CTLS_MVT_v4i32_r(RetVT, Op0);
2413 default: return Register();
2414 }
2415}
2416
2417// FastEmit functions for ISD::CTLZ.
2418
2419Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2420 if (RetVT.SimpleTy != MVT::i32)
2421 return Register();
2422 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2423}
2424
2425Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2426 if (RetVT.SimpleTy != MVT::i64)
2427 return Register();
2428 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2429}
2430
2431Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2432 if (RetVT.SimpleTy != MVT::v8i8)
2433 return Register();
2434 if ((Subtarget->isNeonAvailable())) {
2435 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2436 }
2437 return Register();
2438}
2439
2440Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2441 if (RetVT.SimpleTy != MVT::v16i8)
2442 return Register();
2443 if ((Subtarget->isNeonAvailable())) {
2444 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2445 }
2446 return Register();
2447}
2448
2449Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2450 if (RetVT.SimpleTy != MVT::v4i16)
2451 return Register();
2452 if ((Subtarget->isNeonAvailable())) {
2453 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2454 }
2455 return Register();
2456}
2457
2458Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2459 if (RetVT.SimpleTy != MVT::v8i16)
2460 return Register();
2461 if ((Subtarget->isNeonAvailable())) {
2462 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2463 }
2464 return Register();
2465}
2466
2467Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2468 if (RetVT.SimpleTy != MVT::v2i32)
2469 return Register();
2470 if ((Subtarget->isNeonAvailable())) {
2471 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2472 }
2473 return Register();
2474}
2475
2476Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2477 if (RetVT.SimpleTy != MVT::v4i32)
2478 return Register();
2479 if ((Subtarget->isNeonAvailable())) {
2480 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2481 }
2482 return Register();
2483}
2484
2485Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2486 switch (VT.SimpleTy) {
2487 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2488 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2489 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2490 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2491 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2492 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2493 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2494 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2495 default: return Register();
2496 }
2497}
2498
2499// FastEmit functions for ISD::CTPOP.
2500
2501Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2502 if (RetVT.SimpleTy != MVT::i32)
2503 return Register();
2504 if ((Subtarget->hasCSSC())) {
2505 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2506 }
2507 return Register();
2508}
2509
2510Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2511 if (RetVT.SimpleTy != MVT::i64)
2512 return Register();
2513 if ((Subtarget->hasCSSC())) {
2514 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2515 }
2516 return Register();
2517}
2518
2519Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2520 if (RetVT.SimpleTy != MVT::v8i8)
2521 return Register();
2522 if ((Subtarget->isNeonAvailable())) {
2523 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2524 }
2525 return Register();
2526}
2527
2528Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2529 if (RetVT.SimpleTy != MVT::v16i8)
2530 return Register();
2531 if ((Subtarget->isNeonAvailable())) {
2532 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2533 }
2534 return Register();
2535}
2536
2537Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2538 switch (VT.SimpleTy) {
2539 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2540 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2541 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2542 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2543 default: return Register();
2544 }
2545}
2546
2547// FastEmit functions for ISD::CTTZ.
2548
2549Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2550 if (RetVT.SimpleTy != MVT::i32)
2551 return Register();
2552 if ((Subtarget->hasCSSC())) {
2553 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2554 }
2555 return Register();
2556}
2557
2558Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2559 if (RetVT.SimpleTy != MVT::i64)
2560 return Register();
2561 if ((Subtarget->hasCSSC())) {
2562 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2563 }
2564 return Register();
2565}
2566
2567Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2568 switch (VT.SimpleTy) {
2569 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2570 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2571 default: return Register();
2572 }
2573}
2574
2575// FastEmit functions for ISD::FABS.
2576
2577Register fastEmit_ISD_FABS_MVT_bf16_r(MVT RetVT, Register Op0) {
2578 if (RetVT.SimpleTy != MVT::bf16)
2579 return Register();
2580 if ((Subtarget->hasFullFP16())) {
2581 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2582 }
2583 return Register();
2584}
2585
2586Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2587 if (RetVT.SimpleTy != MVT::f16)
2588 return Register();
2589 if ((Subtarget->hasFullFP16())) {
2590 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2591 }
2592 return Register();
2593}
2594
2595Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2596 if (RetVT.SimpleTy != MVT::f32)
2597 return Register();
2598 if ((Subtarget->hasFPARMv8())) {
2599 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2600 }
2601 return Register();
2602}
2603
2604Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2605 if (RetVT.SimpleTy != MVT::f64)
2606 return Register();
2607 if ((Subtarget->hasFPARMv8())) {
2608 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2609 }
2610 return Register();
2611}
2612
2613Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2614 if (RetVT.SimpleTy != MVT::v4f16)
2615 return Register();
2616 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2617 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2618 }
2619 return Register();
2620}
2621
2622Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2623 if (RetVT.SimpleTy != MVT::v8f16)
2624 return Register();
2625 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2626 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2627 }
2628 return Register();
2629}
2630
2631Register fastEmit_ISD_FABS_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2632 if (RetVT.SimpleTy != MVT::v4bf16)
2633 return Register();
2634 if ((Subtarget->hasFullFP16())) {
2635 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2636 }
2637 return Register();
2638}
2639
2640Register fastEmit_ISD_FABS_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2641 if (RetVT.SimpleTy != MVT::v8bf16)
2642 return Register();
2643 if ((Subtarget->hasFullFP16())) {
2644 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2645 }
2646 return Register();
2647}
2648
2649Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2650 if (RetVT.SimpleTy != MVT::v2f32)
2651 return Register();
2652 if ((Subtarget->isNeonAvailable())) {
2653 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2654 }
2655 return Register();
2656}
2657
2658Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2659 if (RetVT.SimpleTy != MVT::v4f32)
2660 return Register();
2661 if ((Subtarget->isNeonAvailable())) {
2662 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2663 }
2664 return Register();
2665}
2666
2667Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2668 if (RetVT.SimpleTy != MVT::v2f64)
2669 return Register();
2670 if ((Subtarget->isNeonAvailable())) {
2671 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2672 }
2673 return Register();
2674}
2675
2676Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2677 switch (VT.SimpleTy) {
2678 case MVT::bf16: return fastEmit_ISD_FABS_MVT_bf16_r(RetVT, Op0);
2679 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2680 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2681 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2682 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2683 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2684 case MVT::v4bf16: return fastEmit_ISD_FABS_MVT_v4bf16_r(RetVT, Op0);
2685 case MVT::v8bf16: return fastEmit_ISD_FABS_MVT_v8bf16_r(RetVT, Op0);
2686 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2687 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2688 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2689 default: return Register();
2690 }
2691}
2692
2693// FastEmit functions for ISD::FCEIL.
2694
2695Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2696 if (RetVT.SimpleTy != MVT::f16)
2697 return Register();
2698 if ((Subtarget->hasFullFP16())) {
2699 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2700 }
2701 return Register();
2702}
2703
2704Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2705 if (RetVT.SimpleTy != MVT::f32)
2706 return Register();
2707 if ((Subtarget->hasFPARMv8())) {
2708 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2709 }
2710 return Register();
2711}
2712
2713Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2714 if (RetVT.SimpleTy != MVT::f64)
2715 return Register();
2716 if ((Subtarget->hasFPARMv8())) {
2717 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2718 }
2719 return Register();
2720}
2721
2722Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2723 if (RetVT.SimpleTy != MVT::v4f16)
2724 return Register();
2725 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2726 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2727 }
2728 return Register();
2729}
2730
2731Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2732 if (RetVT.SimpleTy != MVT::v8f16)
2733 return Register();
2734 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2736 }
2737 return Register();
2738}
2739
2740Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2741 if (RetVT.SimpleTy != MVT::v2f32)
2742 return Register();
2743 if ((Subtarget->isNeonAvailable())) {
2744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2745 }
2746 return Register();
2747}
2748
2749Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2750 if (RetVT.SimpleTy != MVT::v4f32)
2751 return Register();
2752 if ((Subtarget->isNeonAvailable())) {
2753 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2754 }
2755 return Register();
2756}
2757
2758Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2759 if (RetVT.SimpleTy != MVT::v2f64)
2760 return Register();
2761 if ((Subtarget->isNeonAvailable())) {
2762 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2763 }
2764 return Register();
2765}
2766
2767Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2768 switch (VT.SimpleTy) {
2769 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2770 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2771 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2772 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2773 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2774 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2775 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2776 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2777 default: return Register();
2778 }
2779}
2780
2781// FastEmit functions for ISD::FFLOOR.
2782
2783Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2784 if (RetVT.SimpleTy != MVT::f16)
2785 return Register();
2786 if ((Subtarget->hasFullFP16())) {
2787 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2788 }
2789 return Register();
2790}
2791
2792Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2793 if (RetVT.SimpleTy != MVT::f32)
2794 return Register();
2795 if ((Subtarget->hasFPARMv8())) {
2796 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2797 }
2798 return Register();
2799}
2800
2801Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2802 if (RetVT.SimpleTy != MVT::f64)
2803 return Register();
2804 if ((Subtarget->hasFPARMv8())) {
2805 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2806 }
2807 return Register();
2808}
2809
2810Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2811 if (RetVT.SimpleTy != MVT::v4f16)
2812 return Register();
2813 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2814 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2815 }
2816 return Register();
2817}
2818
2819Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2820 if (RetVT.SimpleTy != MVT::v8f16)
2821 return Register();
2822 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2823 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2824 }
2825 return Register();
2826}
2827
2828Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2829 if (RetVT.SimpleTy != MVT::v2f32)
2830 return Register();
2831 if ((Subtarget->isNeonAvailable())) {
2832 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2833 }
2834 return Register();
2835}
2836
2837Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2838 if (RetVT.SimpleTy != MVT::v4f32)
2839 return Register();
2840 if ((Subtarget->isNeonAvailable())) {
2841 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2842 }
2843 return Register();
2844}
2845
2846Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2847 if (RetVT.SimpleTy != MVT::v2f64)
2848 return Register();
2849 if ((Subtarget->isNeonAvailable())) {
2850 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2851 }
2852 return Register();
2853}
2854
2855Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2856 switch (VT.SimpleTy) {
2857 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2858 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2859 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2860 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2861 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2862 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2863 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2864 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2865 default: return Register();
2866 }
2867}
2868
2869// FastEmit functions for ISD::FNEARBYINT.
2870
2871Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2872 if (RetVT.SimpleTy != MVT::f16)
2873 return Register();
2874 if ((Subtarget->hasFullFP16())) {
2875 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2876 }
2877 return Register();
2878}
2879
2880Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2881 if (RetVT.SimpleTy != MVT::f32)
2882 return Register();
2883 if ((Subtarget->hasFPARMv8())) {
2884 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2885 }
2886 return Register();
2887}
2888
2889Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2890 if (RetVT.SimpleTy != MVT::f64)
2891 return Register();
2892 if ((Subtarget->hasFPARMv8())) {
2893 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2894 }
2895 return Register();
2896}
2897
2898Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2899 if (RetVT.SimpleTy != MVT::v4f16)
2900 return Register();
2901 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2902 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2903 }
2904 return Register();
2905}
2906
2907Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2908 if (RetVT.SimpleTy != MVT::v8f16)
2909 return Register();
2910 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2912 }
2913 return Register();
2914}
2915
2916Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2917 if (RetVT.SimpleTy != MVT::v2f32)
2918 return Register();
2919 if ((Subtarget->isNeonAvailable())) {
2920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2921 }
2922 return Register();
2923}
2924
2925Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2926 if (RetVT.SimpleTy != MVT::v4f32)
2927 return Register();
2928 if ((Subtarget->isNeonAvailable())) {
2929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2930 }
2931 return Register();
2932}
2933
2934Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2935 if (RetVT.SimpleTy != MVT::v2f64)
2936 return Register();
2937 if ((Subtarget->isNeonAvailable())) {
2938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2939 }
2940 return Register();
2941}
2942
2943Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2944 switch (VT.SimpleTy) {
2945 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2946 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2947 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2948 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2949 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2950 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2951 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2952 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2953 default: return Register();
2954 }
2955}
2956
2957// FastEmit functions for ISD::FNEG.
2958
2959Register fastEmit_ISD_FNEG_MVT_bf16_r(MVT RetVT, Register Op0) {
2960 if (RetVT.SimpleTy != MVT::bf16)
2961 return Register();
2962 if ((Subtarget->hasFullFP16())) {
2963 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2964 }
2965 return Register();
2966}
2967
2968Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2969 if (RetVT.SimpleTy != MVT::f16)
2970 return Register();
2971 if ((Subtarget->hasFullFP16())) {
2972 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2973 }
2974 return Register();
2975}
2976
2977Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2978 if (RetVT.SimpleTy != MVT::f32)
2979 return Register();
2980 if ((Subtarget->hasFPARMv8())) {
2981 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2982 }
2983 return Register();
2984}
2985
2986Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2987 if (RetVT.SimpleTy != MVT::f64)
2988 return Register();
2989 if ((Subtarget->hasFPARMv8())) {
2990 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2991 }
2992 return Register();
2993}
2994
2995Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2996 if (RetVT.SimpleTy != MVT::v4f16)
2997 return Register();
2998 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2999 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3000 }
3001 return Register();
3002}
3003
3004Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
3005 if (RetVT.SimpleTy != MVT::v8f16)
3006 return Register();
3007 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3008 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3009 }
3010 return Register();
3011}
3012
3013Register fastEmit_ISD_FNEG_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3014 if (RetVT.SimpleTy != MVT::v4bf16)
3015 return Register();
3016 if ((Subtarget->hasFullFP16())) {
3017 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3018 }
3019 return Register();
3020}
3021
3022Register fastEmit_ISD_FNEG_MVT_v8bf16_r(MVT RetVT, Register Op0) {
3023 if (RetVT.SimpleTy != MVT::v8bf16)
3024 return Register();
3025 if ((Subtarget->hasFullFP16())) {
3026 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3027 }
3028 return Register();
3029}
3030
3031Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
3032 if (RetVT.SimpleTy != MVT::v2f32)
3033 return Register();
3034 if ((Subtarget->isNeonAvailable())) {
3035 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
3041 if (RetVT.SimpleTy != MVT::v4f32)
3042 return Register();
3043 if ((Subtarget->isNeonAvailable())) {
3044 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3045 }
3046 return Register();
3047}
3048
3049Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3050 if (RetVT.SimpleTy != MVT::v2f64)
3051 return Register();
3052 if ((Subtarget->isNeonAvailable())) {
3053 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3054 }
3055 return Register();
3056}
3057
3058Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3059 switch (VT.SimpleTy) {
3060 case MVT::bf16: return fastEmit_ISD_FNEG_MVT_bf16_r(RetVT, Op0);
3061 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3062 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3063 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3064 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3065 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3066 case MVT::v4bf16: return fastEmit_ISD_FNEG_MVT_v4bf16_r(RetVT, Op0);
3067 case MVT::v8bf16: return fastEmit_ISD_FNEG_MVT_v8bf16_r(RetVT, Op0);
3068 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3069 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3070 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3071 default: return Register();
3072 }
3073}
3074
3075// FastEmit functions for ISD::FP_EXTEND.
3076
3077Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3078 if ((Subtarget->hasFPARMv8())) {
3079 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3080 }
3081 return Register();
3082}
3083
3084Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3085 if ((Subtarget->hasFPARMv8())) {
3086 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3087 }
3088 return Register();
3089}
3090
3091Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3092switch (RetVT.SimpleTy) {
3093 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3094 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3095 default: return Register();
3096}
3097}
3098
3099Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3100 if (RetVT.SimpleTy != MVT::f64)
3101 return Register();
3102 if ((Subtarget->hasFPARMv8())) {
3103 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3104 }
3105 return Register();
3106}
3107
3108Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3109 if (RetVT.SimpleTy != MVT::v4f32)
3110 return Register();
3111 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3112}
3113
3114Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3115 if (RetVT.SimpleTy != MVT::v4f32)
3116 return Register();
3117 if ((Subtarget->isNeonAvailable())) {
3118 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3119 }
3120 return Register();
3121}
3122
3123Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3124 if (RetVT.SimpleTy != MVT::v2f64)
3125 return Register();
3126 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3127}
3128
3129Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3130 switch (VT.SimpleTy) {
3131 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3132 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3133 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3134 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3135 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3136 default: return Register();
3137 }
3138}
3139
3140// FastEmit functions for ISD::FP_ROUND.
3141
3142Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3143 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3144 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3145 }
3146 return Register();
3147}
3148
3149Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3150 if ((Subtarget->hasFPARMv8())) {
3151 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3152 }
3153 return Register();
3154}
3155
3156Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3157switch (RetVT.SimpleTy) {
3158 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3159 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3160 default: return Register();
3161}
3162}
3163
3164Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3165 if ((Subtarget->hasFPARMv8())) {
3166 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3167 }
3168 return Register();
3169}
3170
3171Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3172 if ((Subtarget->hasFPARMv8())) {
3173 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3174 }
3175 return Register();
3176}
3177
3178Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3179switch (RetVT.SimpleTy) {
3180 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3181 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3182 default: return Register();
3183}
3184}
3185
3186Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3188}
3189
3190Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3191 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3192 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3193 }
3194 return Register();
3195}
3196
3197Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3198switch (RetVT.SimpleTy) {
3199 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3200 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3201 default: return Register();
3202}
3203}
3204
3205Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3206 if (RetVT.SimpleTy != MVT::v2f32)
3207 return Register();
3208 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3209}
3210
3211Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3212 switch (VT.SimpleTy) {
3213 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3214 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3215 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3216 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3217 default: return Register();
3218 }
3219}
3220
3221// FastEmit functions for ISD::FP_TO_SINT.
3222
3223Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3224 if ((Subtarget->hasFPRCVT())) {
3225 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
3226 }
3227 if ((Subtarget->hasFullFP16())) {
3228 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3229 }
3230 return Register();
3231}
3232
3233Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3234 if ((Subtarget->hasFPRCVT())) {
3235 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
3236 }
3237 if ((Subtarget->hasFullFP16())) {
3238 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3239 }
3240 return Register();
3241}
3242
3243Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3244switch (RetVT.SimpleTy) {
3245 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3246 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3247 default: return Register();
3248}
3249}
3250
3251Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3252 if ((Subtarget->hasFPARMv8())) {
3253 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3254 }
3255 return Register();
3256}
3257
3258Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3259 if ((Subtarget->hasFPRCVT())) {
3260 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
3261 }
3262 if ((Subtarget->hasFPARMv8())) {
3263 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3264 }
3265 return Register();
3266}
3267
3268Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3269switch (RetVT.SimpleTy) {
3270 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3271 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3272 default: return Register();
3273}
3274}
3275
3276Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3277 if ((Subtarget->hasFPRCVT())) {
3278 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
3279 }
3280 if ((Subtarget->hasFPARMv8())) {
3281 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3282 }
3283 return Register();
3284}
3285
3286Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3287 if ((Subtarget->hasFPARMv8())) {
3288 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3289 }
3290 return Register();
3291}
3292
3293Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3294switch (RetVT.SimpleTy) {
3295 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3296 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3297 default: return Register();
3298}
3299}
3300
3301Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3302 if (RetVT.SimpleTy != MVT::v4i16)
3303 return Register();
3304 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3305 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3306 }
3307 return Register();
3308}
3309
3310Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3311 if (RetVT.SimpleTy != MVT::v8i16)
3312 return Register();
3313 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3314 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3315 }
3316 return Register();
3317}
3318
3319Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3320 if (RetVT.SimpleTy != MVT::v2i32)
3321 return Register();
3322 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3323 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3324 }
3325 return Register();
3326}
3327
3328Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3329 if (RetVT.SimpleTy != MVT::v4i32)
3330 return Register();
3331 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3332 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3333 }
3334 return Register();
3335}
3336
3337Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3338 if (RetVT.SimpleTy != MVT::v2i64)
3339 return Register();
3340 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3341 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3342 }
3343 return Register();
3344}
3345
3346Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3347 switch (VT.SimpleTy) {
3348 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3349 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3350 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3351 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3352 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3353 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3354 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3355 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3356 default: return Register();
3357 }
3358}
3359
3360// FastEmit functions for ISD::FP_TO_UINT.
3361
3362Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3363 if ((Subtarget->hasFPRCVT())) {
3364 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
3365 }
3366 if ((Subtarget->hasFullFP16())) {
3367 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3368 }
3369 return Register();
3370}
3371
3372Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3373 if ((Subtarget->hasFPRCVT())) {
3374 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
3375 }
3376 if ((Subtarget->hasFullFP16())) {
3377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3378 }
3379 return Register();
3380}
3381
3382Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3383switch (RetVT.SimpleTy) {
3384 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3385 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3386 default: return Register();
3387}
3388}
3389
3390Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3391 if ((Subtarget->hasFPARMv8())) {
3392 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3393 }
3394 return Register();
3395}
3396
3397Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3398 if ((Subtarget->hasFPRCVT())) {
3399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
3400 }
3401 if ((Subtarget->hasFPARMv8())) {
3402 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3403 }
3404 return Register();
3405}
3406
3407Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3408switch (RetVT.SimpleTy) {
3409 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3410 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3411 default: return Register();
3412}
3413}
3414
3415Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3416 if ((Subtarget->hasFPRCVT())) {
3417 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
3418 }
3419 if ((Subtarget->hasFPARMv8())) {
3420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3421 }
3422 return Register();
3423}
3424
3425Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3426 if ((Subtarget->hasFPARMv8())) {
3427 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3428 }
3429 return Register();
3430}
3431
3432Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3433switch (RetVT.SimpleTy) {
3434 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3435 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3436 default: return Register();
3437}
3438}
3439
3440Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3441 if (RetVT.SimpleTy != MVT::v4i16)
3442 return Register();
3443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3444 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3445 }
3446 return Register();
3447}
3448
3449Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3450 if (RetVT.SimpleTy != MVT::v8i16)
3451 return Register();
3452 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3453 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3454 }
3455 return Register();
3456}
3457
3458Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3459 if (RetVT.SimpleTy != MVT::v2i32)
3460 return Register();
3461 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3462 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3463 }
3464 return Register();
3465}
3466
3467Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3468 if (RetVT.SimpleTy != MVT::v4i32)
3469 return Register();
3470 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3471 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3472 }
3473 return Register();
3474}
3475
3476Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3477 if (RetVT.SimpleTy != MVT::v2i64)
3478 return Register();
3479 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
3480 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3481 }
3482 return Register();
3483}
3484
3485Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3486 switch (VT.SimpleTy) {
3487 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3488 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3489 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3490 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3491 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3492 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3493 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3494 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3495 default: return Register();
3496 }
3497}
3498
3499// FastEmit functions for ISD::FRINT.
3500
3501Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3502 if (RetVT.SimpleTy != MVT::f16)
3503 return Register();
3504 if ((Subtarget->hasFullFP16())) {
3505 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3506 }
3507 return Register();
3508}
3509
3510Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3511 if (RetVT.SimpleTy != MVT::f32)
3512 return Register();
3513 if ((Subtarget->hasFPARMv8())) {
3514 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3515 }
3516 return Register();
3517}
3518
3519Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3520 if (RetVT.SimpleTy != MVT::f64)
3521 return Register();
3522 if ((Subtarget->hasFPARMv8())) {
3523 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3524 }
3525 return Register();
3526}
3527
3528Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3529 if (RetVT.SimpleTy != MVT::v4f16)
3530 return Register();
3531 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3532 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3533 }
3534 return Register();
3535}
3536
3537Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3538 if (RetVT.SimpleTy != MVT::v8f16)
3539 return Register();
3540 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3541 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3542 }
3543 return Register();
3544}
3545
3546Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3547 if (RetVT.SimpleTy != MVT::v2f32)
3548 return Register();
3549 if ((Subtarget->isNeonAvailable())) {
3550 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3551 }
3552 return Register();
3553}
3554
3555Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3556 if (RetVT.SimpleTy != MVT::v4f32)
3557 return Register();
3558 if ((Subtarget->isNeonAvailable())) {
3559 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3560 }
3561 return Register();
3562}
3563
3564Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3565 if (RetVT.SimpleTy != MVT::v2f64)
3566 return Register();
3567 if ((Subtarget->isNeonAvailable())) {
3568 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3569 }
3570 return Register();
3571}
3572
3573Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3574 switch (VT.SimpleTy) {
3575 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3576 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3577 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3578 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3579 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3580 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3581 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3582 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3583 default: return Register();
3584 }
3585}
3586
3587// FastEmit functions for ISD::FROUND.
3588
3589Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3590 if (RetVT.SimpleTy != MVT::f16)
3591 return Register();
3592 if ((Subtarget->hasFullFP16())) {
3593 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3594 }
3595 return Register();
3596}
3597
3598Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3599 if (RetVT.SimpleTy != MVT::f32)
3600 return Register();
3601 if ((Subtarget->hasFPARMv8())) {
3602 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3603 }
3604 return Register();
3605}
3606
3607Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3608 if (RetVT.SimpleTy != MVT::f64)
3609 return Register();
3610 if ((Subtarget->hasFPARMv8())) {
3611 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3612 }
3613 return Register();
3614}
3615
3616Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3617 if (RetVT.SimpleTy != MVT::v4f16)
3618 return Register();
3619 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3620 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3621 }
3622 return Register();
3623}
3624
3625Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3626 if (RetVT.SimpleTy != MVT::v8f16)
3627 return Register();
3628 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3629 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3630 }
3631 return Register();
3632}
3633
3634Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3635 if (RetVT.SimpleTy != MVT::v2f32)
3636 return Register();
3637 if ((Subtarget->isNeonAvailable())) {
3638 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3639 }
3640 return Register();
3641}
3642
3643Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3644 if (RetVT.SimpleTy != MVT::v4f32)
3645 return Register();
3646 if ((Subtarget->isNeonAvailable())) {
3647 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3648 }
3649 return Register();
3650}
3651
3652Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3653 if (RetVT.SimpleTy != MVT::v2f64)
3654 return Register();
3655 if ((Subtarget->isNeonAvailable())) {
3656 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3657 }
3658 return Register();
3659}
3660
3661Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3662 switch (VT.SimpleTy) {
3663 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3664 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3665 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3666 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3667 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3668 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3669 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3670 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3671 default: return Register();
3672 }
3673}
3674
3675// FastEmit functions for ISD::FROUNDEVEN.
3676
3677Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3678 if (RetVT.SimpleTy != MVT::f16)
3679 return Register();
3680 if ((Subtarget->hasFullFP16())) {
3681 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3682 }
3683 return Register();
3684}
3685
3686Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3687 if (RetVT.SimpleTy != MVT::f32)
3688 return Register();
3689 if ((Subtarget->hasFPARMv8())) {
3690 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3691 }
3692 return Register();
3693}
3694
3695Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3696 if (RetVT.SimpleTy != MVT::f64)
3697 return Register();
3698 if ((Subtarget->hasFPARMv8())) {
3699 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3700 }
3701 return Register();
3702}
3703
3704Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3705 if (RetVT.SimpleTy != MVT::v4f16)
3706 return Register();
3707 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3708 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3709 }
3710 return Register();
3711}
3712
3713Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3714 if (RetVT.SimpleTy != MVT::v8f16)
3715 return Register();
3716 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3717 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3718 }
3719 return Register();
3720}
3721
3722Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3723 if (RetVT.SimpleTy != MVT::v2f32)
3724 return Register();
3725 if ((Subtarget->isNeonAvailable())) {
3726 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3727 }
3728 return Register();
3729}
3730
3731Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3732 if (RetVT.SimpleTy != MVT::v4f32)
3733 return Register();
3734 if ((Subtarget->isNeonAvailable())) {
3735 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3736 }
3737 return Register();
3738}
3739
3740Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3741 if (RetVT.SimpleTy != MVT::v2f64)
3742 return Register();
3743 if ((Subtarget->isNeonAvailable())) {
3744 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3745 }
3746 return Register();
3747}
3748
3749Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3750 switch (VT.SimpleTy) {
3751 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3752 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3753 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3754 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3755 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3756 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3757 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3758 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3759 default: return Register();
3760 }
3761}
3762
3763// FastEmit functions for ISD::FSQRT.
3764
3765Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3766 if (RetVT.SimpleTy != MVT::f16)
3767 return Register();
3768 if ((Subtarget->hasFullFP16())) {
3769 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3770 }
3771 return Register();
3772}
3773
3774Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3775 if (RetVT.SimpleTy != MVT::f32)
3776 return Register();
3777 if ((Subtarget->hasFPARMv8())) {
3778 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3779 }
3780 return Register();
3781}
3782
3783Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3784 if (RetVT.SimpleTy != MVT::f64)
3785 return Register();
3786 if ((Subtarget->hasFPARMv8())) {
3787 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3788 }
3789 return Register();
3790}
3791
3792Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3793 if (RetVT.SimpleTy != MVT::v4f16)
3794 return Register();
3795 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3796 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3797 }
3798 return Register();
3799}
3800
3801Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3802 if (RetVT.SimpleTy != MVT::v8f16)
3803 return Register();
3804 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3805 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3806 }
3807 return Register();
3808}
3809
3810Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3811 if (RetVT.SimpleTy != MVT::v2f32)
3812 return Register();
3813 if ((Subtarget->isNeonAvailable())) {
3814 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3815 }
3816 return Register();
3817}
3818
3819Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3820 if (RetVT.SimpleTy != MVT::v4f32)
3821 return Register();
3822 if ((Subtarget->isNeonAvailable())) {
3823 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3824 }
3825 return Register();
3826}
3827
3828Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3829 if (RetVT.SimpleTy != MVT::v2f64)
3830 return Register();
3831 if ((Subtarget->isNeonAvailable())) {
3832 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3833 }
3834 return Register();
3835}
3836
3837Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3838 switch (VT.SimpleTy) {
3839 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3840 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3841 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3842 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3843 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3844 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3845 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3846 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3847 default: return Register();
3848 }
3849}
3850
3851// FastEmit functions for ISD::FTRUNC.
3852
3853Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3854 if (RetVT.SimpleTy != MVT::f16)
3855 return Register();
3856 if ((Subtarget->hasFullFP16())) {
3857 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3858 }
3859 return Register();
3860}
3861
3862Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
3863 if (RetVT.SimpleTy != MVT::f32)
3864 return Register();
3865 if ((Subtarget->hasFPARMv8())) {
3866 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
3867 }
3868 return Register();
3869}
3870
3871Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
3872 if (RetVT.SimpleTy != MVT::f64)
3873 return Register();
3874 if ((Subtarget->hasFPARMv8())) {
3875 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
3876 }
3877 return Register();
3878}
3879
3880Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
3881 if (RetVT.SimpleTy != MVT::v4f16)
3882 return Register();
3883 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3884 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
3885 }
3886 return Register();
3887}
3888
3889Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
3890 if (RetVT.SimpleTy != MVT::v8f16)
3891 return Register();
3892 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3893 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
3894 }
3895 return Register();
3896}
3897
3898Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
3899 if (RetVT.SimpleTy != MVT::v2f32)
3900 return Register();
3901 if ((Subtarget->isNeonAvailable())) {
3902 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
3903 }
3904 return Register();
3905}
3906
3907Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
3908 if (RetVT.SimpleTy != MVT::v4f32)
3909 return Register();
3910 if ((Subtarget->isNeonAvailable())) {
3911 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
3912 }
3913 return Register();
3914}
3915
3916Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
3917 if (RetVT.SimpleTy != MVT::v2f64)
3918 return Register();
3919 if ((Subtarget->isNeonAvailable())) {
3920 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
3921 }
3922 return Register();
3923}
3924
3925Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
3926 switch (VT.SimpleTy) {
3927 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
3928 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
3929 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
3930 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
3931 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
3932 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
3933 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
3934 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
3935 default: return Register();
3936 }
3937}
3938
3939// FastEmit functions for ISD::LLROUND.
3940
3941Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3942 if (RetVT.SimpleTy != MVT::i64)
3943 return Register();
3944 if ((Subtarget->hasFPRCVT())) {
3945 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3946 }
3947 if ((Subtarget->hasFullFP16())) {
3948 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3949 }
3950 return Register();
3951}
3952
3953Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3954 if (RetVT.SimpleTy != MVT::i64)
3955 return Register();
3956 if ((Subtarget->hasFPRCVT())) {
3957 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
3958 }
3959 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
3960}
3961
3962Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3963 if (RetVT.SimpleTy != MVT::i64)
3964 return Register();
3965 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
3966}
3967
3968Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
3969 switch (VT.SimpleTy) {
3970 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
3971 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
3972 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
3973 default: return Register();
3974 }
3975}
3976
3977// FastEmit functions for ISD::LROUND.
3978
3979Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
3980 if ((Subtarget->hasFPRCVT())) {
3981 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSHr, RC: &AArch64::FPR32RegClass, Op0);
3982 }
3983 if ((Subtarget->hasFullFP16())) {
3984 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
3985 }
3986 return Register();
3987}
3988
3989Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
3990 if ((Subtarget->hasFPRCVT())) {
3991 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDHr, RC: &AArch64::FPR64RegClass, Op0);
3992 }
3993 if ((Subtarget->hasFullFP16())) {
3994 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
3995 }
3996 return Register();
3997}
3998
3999Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4000switch (RetVT.SimpleTy) {
4001 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4002 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4003 default: return Register();
4004}
4005}
4006
4007Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4008 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4009}
4010
4011Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4012 if ((Subtarget->hasFPRCVT())) {
4013 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASDSr, RC: &AArch64::FPR64RegClass, Op0);
4014 }
4015 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4016}
4017
4018Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4019switch (RetVT.SimpleTy) {
4020 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4021 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4022 default: return Register();
4023}
4024}
4025
4026Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4027 if ((Subtarget->hasFPRCVT())) {
4028 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASSDr, RC: &AArch64::FPR32RegClass, Op0);
4029 }
4030 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4031}
4032
4033Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4034 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4035}
4036
4037Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4038switch (RetVT.SimpleTy) {
4039 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4040 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4041 default: return Register();
4042}
4043}
4044
4045Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4046 switch (VT.SimpleTy) {
4047 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4048 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4049 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4050 default: return Register();
4051 }
4052}
4053
4054// FastEmit functions for ISD::SINT_TO_FP.
4055
4056Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4057 if ((Subtarget->hasFPRCVT())) {
4058 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4059 }
4060 if ((Subtarget->hasFullFP16())) {
4061 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4062 }
4063 return Register();
4064}
4065
4066Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4067 if ((Subtarget->hasFPARMv8())) {
4068 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4069 }
4070 return Register();
4071}
4072
4073Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4074 if ((Subtarget->hasFPRCVT())) {
4075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4076 }
4077 if ((Subtarget->hasFPARMv8())) {
4078 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4079 }
4080 return Register();
4081}
4082
4083Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4084switch (RetVT.SimpleTy) {
4085 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4086 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4087 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4088 default: return Register();
4089}
4090}
4091
4092Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4093 if ((Subtarget->hasFPRCVT())) {
4094 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4095 }
4096 if ((Subtarget->hasFullFP16())) {
4097 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4098 }
4099 return Register();
4100}
4101
4102Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4103 if ((Subtarget->hasFPRCVT())) {
4104 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4105 }
4106 if ((Subtarget->hasFPARMv8())) {
4107 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4108 }
4109 return Register();
4110}
4111
4112Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4113 if ((Subtarget->hasFPARMv8())) {
4114 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4115 }
4116 return Register();
4117}
4118
4119Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4120switch (RetVT.SimpleTy) {
4121 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4122 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4123 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4124 default: return Register();
4125}
4126}
4127
4128Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4129 if (RetVT.SimpleTy != MVT::v4f16)
4130 return Register();
4131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4132 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4133 }
4134 return Register();
4135}
4136
4137Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4138 if (RetVT.SimpleTy != MVT::v8f16)
4139 return Register();
4140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4141 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4142 }
4143 return Register();
4144}
4145
4146Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4147 if (RetVT.SimpleTy != MVT::v2f32)
4148 return Register();
4149 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4150 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4151 }
4152 return Register();
4153}
4154
4155Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4156 if (RetVT.SimpleTy != MVT::v4f32)
4157 return Register();
4158 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4159 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4160 }
4161 return Register();
4162}
4163
4164Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4165 if (RetVT.SimpleTy != MVT::v2f64)
4166 return Register();
4167 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4168 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4169 }
4170 return Register();
4171}
4172
4173Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4174 switch (VT.SimpleTy) {
4175 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4176 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4177 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4178 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4179 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4180 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4181 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4182 default: return Register();
4183 }
4184}
4185
4186// FastEmit functions for ISD::SPLAT_VECTOR.
4187
4188Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4189 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4190 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4191 }
4192 return Register();
4193}
4194
4195Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4196 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4197 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4198 }
4199 return Register();
4200}
4201
4202Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4203 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4204 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4205 }
4206 return Register();
4207}
4208
4209Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4210switch (RetVT.SimpleTy) {
4211 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4212 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4213 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4214 default: return Register();
4215}
4216}
4217
4218Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4219 if (RetVT.SimpleTy != MVT::nxv2i64)
4220 return Register();
4221 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4222 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4223 }
4224 return Register();
4225}
4226
4227Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4228 switch (VT.SimpleTy) {
4229 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4230 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4231 default: return Register();
4232 }
4233}
4234
4235// FastEmit functions for ISD::STRICT_FCEIL.
4236
4237Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4238 if (RetVT.SimpleTy != MVT::f16)
4239 return Register();
4240 if ((Subtarget->hasFullFP16())) {
4241 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4242 }
4243 return Register();
4244}
4245
4246Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4247 if (RetVT.SimpleTy != MVT::f32)
4248 return Register();
4249 if ((Subtarget->hasFPARMv8())) {
4250 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4251 }
4252 return Register();
4253}
4254
4255Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4256 if (RetVT.SimpleTy != MVT::f64)
4257 return Register();
4258 if ((Subtarget->hasFPARMv8())) {
4259 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4260 }
4261 return Register();
4262}
4263
4264Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4265 if (RetVT.SimpleTy != MVT::v4f16)
4266 return Register();
4267 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4268 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4269 }
4270 return Register();
4271}
4272
4273Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4274 if (RetVT.SimpleTy != MVT::v8f16)
4275 return Register();
4276 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4277 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4278 }
4279 return Register();
4280}
4281
4282Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4283 if (RetVT.SimpleTy != MVT::v2f32)
4284 return Register();
4285 if ((Subtarget->isNeonAvailable())) {
4286 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4287 }
4288 return Register();
4289}
4290
4291Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4292 if (RetVT.SimpleTy != MVT::v4f32)
4293 return Register();
4294 if ((Subtarget->isNeonAvailable())) {
4295 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4296 }
4297 return Register();
4298}
4299
4300Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4301 if (RetVT.SimpleTy != MVT::v2f64)
4302 return Register();
4303 if ((Subtarget->isNeonAvailable())) {
4304 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4305 }
4306 return Register();
4307}
4308
4309Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4310 switch (VT.SimpleTy) {
4311 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4312 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4313 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4314 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4315 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4316 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4317 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4318 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4319 default: return Register();
4320 }
4321}
4322
4323// FastEmit functions for ISD::STRICT_FFLOOR.
4324
4325Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4326 if (RetVT.SimpleTy != MVT::f16)
4327 return Register();
4328 if ((Subtarget->hasFullFP16())) {
4329 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4330 }
4331 return Register();
4332}
4333
4334Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4335 if (RetVT.SimpleTy != MVT::f32)
4336 return Register();
4337 if ((Subtarget->hasFPARMv8())) {
4338 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4339 }
4340 return Register();
4341}
4342
4343Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4344 if (RetVT.SimpleTy != MVT::f64)
4345 return Register();
4346 if ((Subtarget->hasFPARMv8())) {
4347 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4348 }
4349 return Register();
4350}
4351
4352Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4353 if (RetVT.SimpleTy != MVT::v4f16)
4354 return Register();
4355 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4356 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4357 }
4358 return Register();
4359}
4360
4361Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4362 if (RetVT.SimpleTy != MVT::v8f16)
4363 return Register();
4364 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4365 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4366 }
4367 return Register();
4368}
4369
4370Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4371 if (RetVT.SimpleTy != MVT::v2f32)
4372 return Register();
4373 if ((Subtarget->isNeonAvailable())) {
4374 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4375 }
4376 return Register();
4377}
4378
4379Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4380 if (RetVT.SimpleTy != MVT::v4f32)
4381 return Register();
4382 if ((Subtarget->isNeonAvailable())) {
4383 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4384 }
4385 return Register();
4386}
4387
4388Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4389 if (RetVT.SimpleTy != MVT::v2f64)
4390 return Register();
4391 if ((Subtarget->isNeonAvailable())) {
4392 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4393 }
4394 return Register();
4395}
4396
4397Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4398 switch (VT.SimpleTy) {
4399 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4400 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4401 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4402 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4403 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4404 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4405 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4406 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4407 default: return Register();
4408 }
4409}
4410
4411// FastEmit functions for ISD::STRICT_FNEARBYINT.
4412
4413Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4414 if (RetVT.SimpleTy != MVT::f16)
4415 return Register();
4416 if ((Subtarget->hasFullFP16())) {
4417 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4418 }
4419 return Register();
4420}
4421
4422Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4423 if (RetVT.SimpleTy != MVT::f32)
4424 return Register();
4425 if ((Subtarget->hasFPARMv8())) {
4426 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4427 }
4428 return Register();
4429}
4430
4431Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4432 if (RetVT.SimpleTy != MVT::f64)
4433 return Register();
4434 if ((Subtarget->hasFPARMv8())) {
4435 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4436 }
4437 return Register();
4438}
4439
4440Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4441 if (RetVT.SimpleTy != MVT::v4f16)
4442 return Register();
4443 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4444 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4445 }
4446 return Register();
4447}
4448
4449Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4450 if (RetVT.SimpleTy != MVT::v8f16)
4451 return Register();
4452 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4453 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4454 }
4455 return Register();
4456}
4457
4458Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4459 if (RetVT.SimpleTy != MVT::v2f32)
4460 return Register();
4461 if ((Subtarget->isNeonAvailable())) {
4462 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4463 }
4464 return Register();
4465}
4466
4467Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4468 if (RetVT.SimpleTy != MVT::v4f32)
4469 return Register();
4470 if ((Subtarget->isNeonAvailable())) {
4471 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4472 }
4473 return Register();
4474}
4475
4476Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4477 if (RetVT.SimpleTy != MVT::v2f64)
4478 return Register();
4479 if ((Subtarget->isNeonAvailable())) {
4480 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4481 }
4482 return Register();
4483}
4484
4485Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4486 switch (VT.SimpleTy) {
4487 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4488 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4489 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4490 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4491 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4492 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4493 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4494 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4495 default: return Register();
4496 }
4497}
4498
4499// FastEmit functions for ISD::STRICT_FP_EXTEND.
4500
4501Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4502 if ((Subtarget->hasFPARMv8())) {
4503 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4504 }
4505 return Register();
4506}
4507
4508Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4509 if ((Subtarget->hasFPARMv8())) {
4510 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4511 }
4512 return Register();
4513}
4514
4515Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4516switch (RetVT.SimpleTy) {
4517 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4518 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4519 default: return Register();
4520}
4521}
4522
4523Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4524 if (RetVT.SimpleTy != MVT::f64)
4525 return Register();
4526 if ((Subtarget->hasFPARMv8())) {
4527 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4528 }
4529 return Register();
4530}
4531
4532Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4533 if (RetVT.SimpleTy != MVT::v4f32)
4534 return Register();
4535 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4536}
4537
4538Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4539 if (RetVT.SimpleTy != MVT::v4f32)
4540 return Register();
4541 if ((Subtarget->isNeonAvailable())) {
4542 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4543 }
4544 return Register();
4545}
4546
4547Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4548 if (RetVT.SimpleTy != MVT::v2f64)
4549 return Register();
4550 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4551}
4552
4553Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4554 switch (VT.SimpleTy) {
4555 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4556 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4557 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4558 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4559 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4560 default: return Register();
4561 }
4562}
4563
4564// FastEmit functions for ISD::STRICT_FP_ROUND.
4565
4566Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4567 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4568 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4569 }
4570 return Register();
4571}
4572
4573Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4574 if ((Subtarget->hasFPARMv8())) {
4575 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4576 }
4577 return Register();
4578}
4579
4580Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4581switch (RetVT.SimpleTy) {
4582 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4583 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4584 default: return Register();
4585}
4586}
4587
4588Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4589 if ((Subtarget->hasFPARMv8())) {
4590 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4591 }
4592 return Register();
4593}
4594
4595Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4596 if ((Subtarget->hasFPARMv8())) {
4597 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4598 }
4599 return Register();
4600}
4601
4602Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4603switch (RetVT.SimpleTy) {
4604 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4605 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4606 default: return Register();
4607}
4608}
4609
4610Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4611 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4612}
4613
4614Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4615 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4616 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4617 }
4618 return Register();
4619}
4620
4621Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4622switch (RetVT.SimpleTy) {
4623 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4624 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4625 default: return Register();
4626}
4627}
4628
4629Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4630 if (RetVT.SimpleTy != MVT::v2f32)
4631 return Register();
4632 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4633}
4634
4635Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4636 switch (VT.SimpleTy) {
4637 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4638 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4639 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4640 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4641 default: return Register();
4642 }
4643}
4644
4645// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4646
4647Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4648 if ((Subtarget->hasFPRCVT())) {
4649 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSHr, RC: &AArch64::FPR32RegClass, Op0);
4650 }
4651 if ((Subtarget->hasFullFP16())) {
4652 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4653 }
4654 return Register();
4655}
4656
4657Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4658 if ((Subtarget->hasFPRCVT())) {
4659 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDHr, RC: &AArch64::FPR64RegClass, Op0);
4660 }
4661 if ((Subtarget->hasFullFP16())) {
4662 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4663 }
4664 return Register();
4665}
4666
4667Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4668switch (RetVT.SimpleTy) {
4669 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4670 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4671 default: return Register();
4672}
4673}
4674
4675Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4676 if ((Subtarget->hasFPARMv8())) {
4677 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4678 }
4679 return Register();
4680}
4681
4682Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4683 if ((Subtarget->hasFPRCVT())) {
4684 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSDSr, RC: &AArch64::FPR64RegClass, Op0);
4685 }
4686 if ((Subtarget->hasFPARMv8())) {
4687 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4688 }
4689 return Register();
4690}
4691
4692Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4693switch (RetVT.SimpleTy) {
4694 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4695 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4696 default: return Register();
4697}
4698}
4699
4700Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4701 if ((Subtarget->hasFPRCVT())) {
4702 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSSDr, RC: &AArch64::FPR32RegClass, Op0);
4703 }
4704 if ((Subtarget->hasFPARMv8())) {
4705 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4706 }
4707 return Register();
4708}
4709
4710Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4711 if ((Subtarget->hasFPARMv8())) {
4712 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4713 }
4714 return Register();
4715}
4716
4717Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4718switch (RetVT.SimpleTy) {
4719 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4720 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4721 default: return Register();
4722}
4723}
4724
4725Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4726 if (RetVT.SimpleTy != MVT::v4i16)
4727 return Register();
4728 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4729 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4730 }
4731 return Register();
4732}
4733
4734Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4735 if (RetVT.SimpleTy != MVT::v8i16)
4736 return Register();
4737 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4738 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4739 }
4740 return Register();
4741}
4742
4743Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4744 if (RetVT.SimpleTy != MVT::v2i32)
4745 return Register();
4746 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4747 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4748 }
4749 return Register();
4750}
4751
4752Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4753 if (RetVT.SimpleTy != MVT::v4i32)
4754 return Register();
4755 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4756 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4757 }
4758 return Register();
4759}
4760
4761Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4762 if (RetVT.SimpleTy != MVT::v2i64)
4763 return Register();
4764 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4765 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4766 }
4767 return Register();
4768}
4769
4770Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4771 switch (VT.SimpleTy) {
4772 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4773 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4774 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4775 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4776 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4777 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4778 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4779 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4780 default: return Register();
4781 }
4782}
4783
4784// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4785
4786Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4787 if ((Subtarget->hasFPRCVT())) {
4788 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSHr, RC: &AArch64::FPR32RegClass, Op0);
4789 }
4790 if ((Subtarget->hasFullFP16())) {
4791 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4792 }
4793 return Register();
4794}
4795
4796Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4797 if ((Subtarget->hasFPRCVT())) {
4798 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDHr, RC: &AArch64::FPR64RegClass, Op0);
4799 }
4800 if ((Subtarget->hasFullFP16())) {
4801 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4802 }
4803 return Register();
4804}
4805
4806Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4807switch (RetVT.SimpleTy) {
4808 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4809 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4810 default: return Register();
4811}
4812}
4813
4814Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4815 if ((Subtarget->hasFPARMv8())) {
4816 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4817 }
4818 return Register();
4819}
4820
4821Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4822 if ((Subtarget->hasFPRCVT())) {
4823 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUDSr, RC: &AArch64::FPR64RegClass, Op0);
4824 }
4825 if ((Subtarget->hasFPARMv8())) {
4826 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4827 }
4828 return Register();
4829}
4830
4831Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4832switch (RetVT.SimpleTy) {
4833 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4834 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4835 default: return Register();
4836}
4837}
4838
4839Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4840 if ((Subtarget->hasFPRCVT())) {
4841 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUSDr, RC: &AArch64::FPR32RegClass, Op0);
4842 }
4843 if ((Subtarget->hasFPARMv8())) {
4844 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4845 }
4846 return Register();
4847}
4848
4849Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4850 if ((Subtarget->hasFPARMv8())) {
4851 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4852 }
4853 return Register();
4854}
4855
4856Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4857switch (RetVT.SimpleTy) {
4858 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4859 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4860 default: return Register();
4861}
4862}
4863
4864Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4865 if (RetVT.SimpleTy != MVT::v4i16)
4866 return Register();
4867 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4868 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4869 }
4870 return Register();
4871}
4872
4873Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4874 if (RetVT.SimpleTy != MVT::v8i16)
4875 return Register();
4876 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4877 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4878 }
4879 return Register();
4880}
4881
4882Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4883 if (RetVT.SimpleTy != MVT::v2i32)
4884 return Register();
4885 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4886 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4887 }
4888 return Register();
4889}
4890
4891Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4892 if (RetVT.SimpleTy != MVT::v4i32)
4893 return Register();
4894 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4895 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4896 }
4897 return Register();
4898}
4899
4900Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4901 if (RetVT.SimpleTy != MVT::v2i64)
4902 return Register();
4903 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
4904 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4905 }
4906 return Register();
4907}
4908
4909Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
4910 switch (VT.SimpleTy) {
4911 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4912 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4913 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4914 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4915 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4916 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4917 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4918 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4919 default: return Register();
4920 }
4921}
4922
4923// FastEmit functions for ISD::STRICT_FRINT.
4924
4925Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
4926 if (RetVT.SimpleTy != MVT::f16)
4927 return Register();
4928 if ((Subtarget->hasFullFP16())) {
4929 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4930 }
4931 return Register();
4932}
4933
4934Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
4935 if (RetVT.SimpleTy != MVT::f32)
4936 return Register();
4937 if ((Subtarget->hasFPARMv8())) {
4938 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4939 }
4940 return Register();
4941}
4942
4943Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
4944 if (RetVT.SimpleTy != MVT::f64)
4945 return Register();
4946 if ((Subtarget->hasFPARMv8())) {
4947 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4948 }
4949 return Register();
4950}
4951
4952Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4953 if (RetVT.SimpleTy != MVT::v4f16)
4954 return Register();
4955 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4956 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4957 }
4958 return Register();
4959}
4960
4961Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4962 if (RetVT.SimpleTy != MVT::v8f16)
4963 return Register();
4964 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4965 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4966 }
4967 return Register();
4968}
4969
4970Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4971 if (RetVT.SimpleTy != MVT::v2f32)
4972 return Register();
4973 if ((Subtarget->isNeonAvailable())) {
4974 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4975 }
4976 return Register();
4977}
4978
4979Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4980 if (RetVT.SimpleTy != MVT::v4f32)
4981 return Register();
4982 if ((Subtarget->isNeonAvailable())) {
4983 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4984 }
4985 return Register();
4986}
4987
4988Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4989 if (RetVT.SimpleTy != MVT::v2f64)
4990 return Register();
4991 if ((Subtarget->isNeonAvailable())) {
4992 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
4993 }
4994 return Register();
4995}
4996
4997Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
4998 switch (VT.SimpleTy) {
4999 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5000 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5001 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5002 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5003 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5004 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5005 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5006 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5007 default: return Register();
5008 }
5009}
5010
5011// FastEmit functions for ISD::STRICT_FROUND.
5012
5013Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5014 if (RetVT.SimpleTy != MVT::f16)
5015 return Register();
5016 if ((Subtarget->hasFullFP16())) {
5017 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5018 }
5019 return Register();
5020}
5021
5022Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5023 if (RetVT.SimpleTy != MVT::f32)
5024 return Register();
5025 if ((Subtarget->hasFPARMv8())) {
5026 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5027 }
5028 return Register();
5029}
5030
5031Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5032 if (RetVT.SimpleTy != MVT::f64)
5033 return Register();
5034 if ((Subtarget->hasFPARMv8())) {
5035 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5036 }
5037 return Register();
5038}
5039
5040Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5041 if (RetVT.SimpleTy != MVT::v4f16)
5042 return Register();
5043 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5044 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5045 }
5046 return Register();
5047}
5048
5049Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5050 if (RetVT.SimpleTy != MVT::v8f16)
5051 return Register();
5052 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5053 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5054 }
5055 return Register();
5056}
5057
5058Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5059 if (RetVT.SimpleTy != MVT::v2f32)
5060 return Register();
5061 if ((Subtarget->isNeonAvailable())) {
5062 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5063 }
5064 return Register();
5065}
5066
5067Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5068 if (RetVT.SimpleTy != MVT::v4f32)
5069 return Register();
5070 if ((Subtarget->isNeonAvailable())) {
5071 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5072 }
5073 return Register();
5074}
5075
5076Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5077 if (RetVT.SimpleTy != MVT::v2f64)
5078 return Register();
5079 if ((Subtarget->isNeonAvailable())) {
5080 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5081 }
5082 return Register();
5083}
5084
5085Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5086 switch (VT.SimpleTy) {
5087 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5088 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5089 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5090 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5091 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5092 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5093 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5094 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5095 default: return Register();
5096 }
5097}
5098
5099// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5100
5101Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5102 if (RetVT.SimpleTy != MVT::f16)
5103 return Register();
5104 if ((Subtarget->hasFullFP16())) {
5105 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5106 }
5107 return Register();
5108}
5109
5110Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5111 if (RetVT.SimpleTy != MVT::f32)
5112 return Register();
5113 if ((Subtarget->hasFPARMv8())) {
5114 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5115 }
5116 return Register();
5117}
5118
5119Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5120 if (RetVT.SimpleTy != MVT::f64)
5121 return Register();
5122 if ((Subtarget->hasFPARMv8())) {
5123 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5124 }
5125 return Register();
5126}
5127
5128Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5129 if (RetVT.SimpleTy != MVT::v4f16)
5130 return Register();
5131 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5132 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5133 }
5134 return Register();
5135}
5136
5137Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5138 if (RetVT.SimpleTy != MVT::v8f16)
5139 return Register();
5140 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5141 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5142 }
5143 return Register();
5144}
5145
5146Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5147 if (RetVT.SimpleTy != MVT::v2f32)
5148 return Register();
5149 if ((Subtarget->isNeonAvailable())) {
5150 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5151 }
5152 return Register();
5153}
5154
5155Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5156 if (RetVT.SimpleTy != MVT::v4f32)
5157 return Register();
5158 if ((Subtarget->isNeonAvailable())) {
5159 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5160 }
5161 return Register();
5162}
5163
5164Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5165 if (RetVT.SimpleTy != MVT::v2f64)
5166 return Register();
5167 if ((Subtarget->isNeonAvailable())) {
5168 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5169 }
5170 return Register();
5171}
5172
5173Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5174 switch (VT.SimpleTy) {
5175 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5176 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5177 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5178 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5179 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5180 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5181 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5182 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5183 default: return Register();
5184 }
5185}
5186
5187// FastEmit functions for ISD::STRICT_FSQRT.
5188
5189Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5190 if (RetVT.SimpleTy != MVT::f16)
5191 return Register();
5192 if ((Subtarget->hasFullFP16())) {
5193 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5194 }
5195 return Register();
5196}
5197
5198Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5199 if (RetVT.SimpleTy != MVT::f32)
5200 return Register();
5201 if ((Subtarget->hasFPARMv8())) {
5202 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5203 }
5204 return Register();
5205}
5206
5207Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5208 if (RetVT.SimpleTy != MVT::f64)
5209 return Register();
5210 if ((Subtarget->hasFPARMv8())) {
5211 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5212 }
5213 return Register();
5214}
5215
5216Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5217 if (RetVT.SimpleTy != MVT::v4f16)
5218 return Register();
5219 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5220 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5221 }
5222 return Register();
5223}
5224
5225Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5226 if (RetVT.SimpleTy != MVT::v8f16)
5227 return Register();
5228 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5229 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5230 }
5231 return Register();
5232}
5233
5234Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5235 if (RetVT.SimpleTy != MVT::v2f32)
5236 return Register();
5237 if ((Subtarget->isNeonAvailable())) {
5238 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5239 }
5240 return Register();
5241}
5242
5243Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5244 if (RetVT.SimpleTy != MVT::v4f32)
5245 return Register();
5246 if ((Subtarget->isNeonAvailable())) {
5247 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5248 }
5249 return Register();
5250}
5251
5252Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5253 if (RetVT.SimpleTy != MVT::v2f64)
5254 return Register();
5255 if ((Subtarget->isNeonAvailable())) {
5256 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5257 }
5258 return Register();
5259}
5260
5261Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5262 switch (VT.SimpleTy) {
5263 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5264 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5265 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5266 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5267 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5268 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5269 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5270 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5271 default: return Register();
5272 }
5273}
5274
5275// FastEmit functions for ISD::STRICT_FTRUNC.
5276
5277Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5278 if (RetVT.SimpleTy != MVT::f16)
5279 return Register();
5280 if ((Subtarget->hasFullFP16())) {
5281 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5282 }
5283 return Register();
5284}
5285
5286Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5287 if (RetVT.SimpleTy != MVT::f32)
5288 return Register();
5289 if ((Subtarget->hasFPARMv8())) {
5290 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5291 }
5292 return Register();
5293}
5294
5295Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5296 if (RetVT.SimpleTy != MVT::f64)
5297 return Register();
5298 if ((Subtarget->hasFPARMv8())) {
5299 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5300 }
5301 return Register();
5302}
5303
5304Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5305 if (RetVT.SimpleTy != MVT::v4f16)
5306 return Register();
5307 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5308 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5309 }
5310 return Register();
5311}
5312
5313Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5314 if (RetVT.SimpleTy != MVT::v8f16)
5315 return Register();
5316 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5317 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5318 }
5319 return Register();
5320}
5321
5322Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5323 if (RetVT.SimpleTy != MVT::v2f32)
5324 return Register();
5325 if ((Subtarget->isNeonAvailable())) {
5326 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5327 }
5328 return Register();
5329}
5330
5331Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5332 if (RetVT.SimpleTy != MVT::v4f32)
5333 return Register();
5334 if ((Subtarget->isNeonAvailable())) {
5335 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5336 }
5337 return Register();
5338}
5339
5340Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5341 if (RetVT.SimpleTy != MVT::v2f64)
5342 return Register();
5343 if ((Subtarget->isNeonAvailable())) {
5344 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5345 }
5346 return Register();
5347}
5348
5349Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5350 switch (VT.SimpleTy) {
5351 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5352 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5353 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5354 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5355 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5356 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5357 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5358 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5359 default: return Register();
5360 }
5361}
5362
5363// FastEmit functions for ISD::STRICT_LLROUND.
5364
5365Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5366 if (RetVT.SimpleTy != MVT::i64)
5367 return Register();
5368 if ((Subtarget->hasFullFP16())) {
5369 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5370 }
5371 return Register();
5372}
5373
5374Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5375 if (RetVT.SimpleTy != MVT::i64)
5376 return Register();
5377 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5378}
5379
5380Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5381 if (RetVT.SimpleTy != MVT::i64)
5382 return Register();
5383 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5384}
5385
5386Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5387 switch (VT.SimpleTy) {
5388 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5389 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5390 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5391 default: return Register();
5392 }
5393}
5394
5395// FastEmit functions for ISD::STRICT_LROUND.
5396
5397Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5398 if ((Subtarget->hasFullFP16())) {
5399 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5400 }
5401 return Register();
5402}
5403
5404Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5405 if ((Subtarget->hasFullFP16())) {
5406 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5407 }
5408 return Register();
5409}
5410
5411Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5412switch (RetVT.SimpleTy) {
5413 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5414 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5415 default: return Register();
5416}
5417}
5418
5419Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5420 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5421}
5422
5423Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5424 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5425}
5426
5427Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5428switch (RetVT.SimpleTy) {
5429 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5430 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5431 default: return Register();
5432}
5433}
5434
5435Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5436 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5437}
5438
5439Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5440 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5441}
5442
5443Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5444switch (RetVT.SimpleTy) {
5445 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5446 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5447 default: return Register();
5448}
5449}
5450
5451Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5452 switch (VT.SimpleTy) {
5453 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5454 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5455 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5456 default: return Register();
5457 }
5458}
5459
5460// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5461
5462Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5463 if ((Subtarget->hasFPRCVT())) {
5464 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5465 }
5466 if ((Subtarget->hasFullFP16())) {
5467 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5468 }
5469 return Register();
5470}
5471
5472Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5473 if ((Subtarget->hasFPARMv8())) {
5474 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5475 }
5476 return Register();
5477}
5478
5479Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5480 if ((Subtarget->hasFPRCVT())) {
5481 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5482 }
5483 if ((Subtarget->hasFPARMv8())) {
5484 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5485 }
5486 return Register();
5487}
5488
5489Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5490switch (RetVT.SimpleTy) {
5491 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5492 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5493 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5494 default: return Register();
5495}
5496}
5497
5498Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5499 if ((Subtarget->hasFPRCVT())) {
5500 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5501 }
5502 if ((Subtarget->hasFullFP16())) {
5503 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5504 }
5505 return Register();
5506}
5507
5508Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5509 if ((Subtarget->hasFPRCVT())) {
5510 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5511 }
5512 if ((Subtarget->hasFPARMv8())) {
5513 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5514 }
5515 return Register();
5516}
5517
5518Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5519 if ((Subtarget->hasFPARMv8())) {
5520 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5521 }
5522 return Register();
5523}
5524
5525Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5526switch (RetVT.SimpleTy) {
5527 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5528 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5529 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5530 default: return Register();
5531}
5532}
5533
5534Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5535 if (RetVT.SimpleTy != MVT::v4f16)
5536 return Register();
5537 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5538 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5539 }
5540 return Register();
5541}
5542
5543Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5544 if (RetVT.SimpleTy != MVT::v8f16)
5545 return Register();
5546 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5547 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5548 }
5549 return Register();
5550}
5551
5552Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5553 if (RetVT.SimpleTy != MVT::v2f32)
5554 return Register();
5555 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5556 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5557 }
5558 return Register();
5559}
5560
5561Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5562 if (RetVT.SimpleTy != MVT::v4f32)
5563 return Register();
5564 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5565 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5566 }
5567 return Register();
5568}
5569
5570Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5571 if (RetVT.SimpleTy != MVT::v2f64)
5572 return Register();
5573 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5574 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5575 }
5576 return Register();
5577}
5578
5579Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5580 switch (VT.SimpleTy) {
5581 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5582 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5583 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5584 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5585 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5586 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5587 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5588 default: return Register();
5589 }
5590}
5591
5592// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5593
5594Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5595 if ((Subtarget->hasFPRCVT())) {
5596 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5597 }
5598 if ((Subtarget->hasFullFP16())) {
5599 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5600 }
5601 return Register();
5602}
5603
5604Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5605 if ((Subtarget->hasFPARMv8())) {
5606 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5607 }
5608 return Register();
5609}
5610
5611Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5612 if ((Subtarget->hasFPRCVT())) {
5613 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5614 }
5615 if ((Subtarget->hasFPARMv8())) {
5616 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5617 }
5618 return Register();
5619}
5620
5621Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5622switch (RetVT.SimpleTy) {
5623 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5624 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5625 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5626 default: return Register();
5627}
5628}
5629
5630Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5631 if ((Subtarget->hasFPRCVT())) {
5632 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5633 }
5634 if ((Subtarget->hasFullFP16())) {
5635 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5636 }
5637 return Register();
5638}
5639
5640Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5641 if ((Subtarget->hasFPRCVT())) {
5642 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5643 }
5644 if ((Subtarget->hasFPARMv8())) {
5645 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5646 }
5647 return Register();
5648}
5649
5650Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5651 if ((Subtarget->hasFPARMv8())) {
5652 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5653 }
5654 return Register();
5655}
5656
5657Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5658switch (RetVT.SimpleTy) {
5659 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5660 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5661 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5662 default: return Register();
5663}
5664}
5665
5666Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5667 if (RetVT.SimpleTy != MVT::v4f16)
5668 return Register();
5669 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5670 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5671 }
5672 return Register();
5673}
5674
5675Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5676 if (RetVT.SimpleTy != MVT::v8f16)
5677 return Register();
5678 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5679 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5680 }
5681 return Register();
5682}
5683
5684Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5685 if (RetVT.SimpleTy != MVT::v2f32)
5686 return Register();
5687 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5688 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5689 }
5690 return Register();
5691}
5692
5693Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5694 if (RetVT.SimpleTy != MVT::v4f32)
5695 return Register();
5696 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5697 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5698 }
5699 return Register();
5700}
5701
5702Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5703 if (RetVT.SimpleTy != MVT::v2f64)
5704 return Register();
5705 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5706 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5707 }
5708 return Register();
5709}
5710
5711Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5712 switch (VT.SimpleTy) {
5713 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5714 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5715 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5716 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5717 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5718 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5719 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5720 default: return Register();
5721 }
5722}
5723
5724// FastEmit functions for ISD::TRUNCATE.
5725
5726Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5727 if (RetVT.SimpleTy != MVT::i32)
5728 return Register();
5729 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5730}
5731
5732Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5733 if (RetVT.SimpleTy != MVT::v8i8)
5734 return Register();
5735 if ((Subtarget->isNeonAvailable())) {
5736 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5737 }
5738 return Register();
5739}
5740
5741Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5742 if (RetVT.SimpleTy != MVT::v4i16)
5743 return Register();
5744 if ((Subtarget->isNeonAvailable())) {
5745 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5746 }
5747 return Register();
5748}
5749
5750Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5751 if (RetVT.SimpleTy != MVT::v2i32)
5752 return Register();
5753 if ((Subtarget->isNeonAvailable())) {
5754 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5755 }
5756 return Register();
5757}
5758
5759Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5760 switch (VT.SimpleTy) {
5761 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5762 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5763 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5764 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5765 default: return Register();
5766 }
5767}
5768
5769// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5770
5771Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5772 if (RetVT.SimpleTy != MVT::v8i8)
5773 return Register();
5774 if ((Subtarget->isNeonAvailable())) {
5775 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5776 }
5777 return Register();
5778}
5779
5780Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5781 if (RetVT.SimpleTy != MVT::v4i16)
5782 return Register();
5783 if ((Subtarget->isNeonAvailable())) {
5784 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5785 }
5786 return Register();
5787}
5788
5789Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5790 if (RetVT.SimpleTy != MVT::v2i32)
5791 return Register();
5792 if ((Subtarget->isNeonAvailable())) {
5793 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5794 }
5795 return Register();
5796}
5797
5798Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5799 switch (VT.SimpleTy) {
5800 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5801 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5802 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5803 default: return Register();
5804 }
5805}
5806
5807// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5808
5809Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5810 if (RetVT.SimpleTy != MVT::v8i8)
5811 return Register();
5812 if ((Subtarget->isNeonAvailable())) {
5813 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5814 }
5815 return Register();
5816}
5817
5818Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5819 if (RetVT.SimpleTy != MVT::v4i16)
5820 return Register();
5821 if ((Subtarget->isNeonAvailable())) {
5822 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5823 }
5824 return Register();
5825}
5826
5827Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5828 if (RetVT.SimpleTy != MVT::v2i32)
5829 return Register();
5830 if ((Subtarget->isNeonAvailable())) {
5831 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5837 switch (VT.SimpleTy) {
5838 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5839 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5840 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5841 default: return Register();
5842 }
5843}
5844
5845// FastEmit functions for ISD::TRUNCATE_USAT_U.
5846
5847Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5848 if (RetVT.SimpleTy != MVT::v8i8)
5849 return Register();
5850 if ((Subtarget->isNeonAvailable())) {
5851 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5852 }
5853 return Register();
5854}
5855
5856Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5857 if (RetVT.SimpleTy != MVT::v4i16)
5858 return Register();
5859 if ((Subtarget->isNeonAvailable())) {
5860 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5861 }
5862 return Register();
5863}
5864
5865Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5866 if (RetVT.SimpleTy != MVT::v2i32)
5867 return Register();
5868 if ((Subtarget->isNeonAvailable())) {
5869 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5870 }
5871 return Register();
5872}
5873
5874Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5875 switch (VT.SimpleTy) {
5876 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5877 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5878 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5879 default: return Register();
5880 }
5881}
5882
5883// FastEmit functions for ISD::UINT_TO_FP.
5884
5885Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5886 if ((Subtarget->hasFPRCVT())) {
5887 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5888 }
5889 if ((Subtarget->hasFullFP16())) {
5890 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5891 }
5892 return Register();
5893}
5894
5895Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5896 if ((Subtarget->hasFPARMv8())) {
5897 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5898 }
5899 return Register();
5900}
5901
5902Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5903 if ((Subtarget->hasFPRCVT())) {
5904 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5905 }
5906 if ((Subtarget->hasFPARMv8())) {
5907 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5908 }
5909 return Register();
5910}
5911
5912Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5913switch (RetVT.SimpleTy) {
5914 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5915 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5916 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5917 default: return Register();
5918}
5919}
5920
5921Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5922 if ((Subtarget->hasFPRCVT())) {
5923 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5924 }
5925 if ((Subtarget->hasFullFP16())) {
5926 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5927 }
5928 return Register();
5929}
5930
5931Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5932 if ((Subtarget->hasFPRCVT())) {
5933 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5934 }
5935 if ((Subtarget->hasFPARMv8())) {
5936 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5937 }
5938 return Register();
5939}
5940
5941Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5942 if ((Subtarget->hasFPARMv8())) {
5943 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5944 }
5945 return Register();
5946}
5947
5948Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5949switch (RetVT.SimpleTy) {
5950 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5951 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5952 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5953 default: return Register();
5954}
5955}
5956
5957Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5958 if (RetVT.SimpleTy != MVT::v4f16)
5959 return Register();
5960 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5961 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5962 }
5963 return Register();
5964}
5965
5966Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5967 if (RetVT.SimpleTy != MVT::v8f16)
5968 return Register();
5969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5970 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5971 }
5972 return Register();
5973}
5974
5975Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5976 if (RetVT.SimpleTy != MVT::v2f32)
5977 return Register();
5978 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5979 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5980 }
5981 return Register();
5982}
5983
5984Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5985 if (RetVT.SimpleTy != MVT::v4f32)
5986 return Register();
5987 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5988 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5989 }
5990 return Register();
5991}
5992
5993Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5994 if (RetVT.SimpleTy != MVT::v2f64)
5995 return Register();
5996 if ((Subtarget->isNeonAvailable() || (Subtarget->hasNEON() && Subtarget->hasFPRCVT()))) {
5997 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5998 }
5999 return Register();
6000}
6001
6002Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6003 switch (VT.SimpleTy) {
6004 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6005 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6006 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6007 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6008 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6009 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6010 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6011 default: return Register();
6012 }
6013}
6014
6015// FastEmit functions for ISD::VECREDUCE_ADD.
6016
6017Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6018 if (RetVT.SimpleTy != MVT::i8)
6019 return Register();
6020 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6021}
6022
6023Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6024 if (RetVT.SimpleTy != MVT::i8)
6025 return Register();
6026 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6027}
6028
6029Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6030 if (RetVT.SimpleTy != MVT::i16)
6031 return Register();
6032 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6033}
6034
6035Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6036 if (RetVT.SimpleTy != MVT::i16)
6037 return Register();
6038 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6039}
6040
6041Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6042 if (RetVT.SimpleTy != MVT::i32)
6043 return Register();
6044 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6045}
6046
6047Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6048 if (RetVT.SimpleTy != MVT::i64)
6049 return Register();
6050 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6051}
6052
6053Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6054 switch (VT.SimpleTy) {
6055 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6056 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6057 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6058 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6059 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6060 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6061 default: return Register();
6062 }
6063}
6064
6065// FastEmit functions for ISD::VECREDUCE_FADD.
6066
6067Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6068 if (RetVT.SimpleTy != MVT::f32)
6069 return Register();
6070 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6071}
6072
6073Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6074 if (RetVT.SimpleTy != MVT::f64)
6075 return Register();
6076 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6077}
6078
6079Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6080 switch (VT.SimpleTy) {
6081 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6082 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6083 default: return Register();
6084 }
6085}
6086
6087// FastEmit functions for ISD::VECREDUCE_FMAX.
6088
6089Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6090 if (RetVT.SimpleTy != MVT::f16)
6091 return Register();
6092 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6093 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6094 }
6095 return Register();
6096}
6097
6098Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6099 if (RetVT.SimpleTy != MVT::f16)
6100 return Register();
6101 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6102 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6103 }
6104 return Register();
6105}
6106
6107Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6108 if (RetVT.SimpleTy != MVT::f32)
6109 return Register();
6110 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6111}
6112
6113Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6114 if (RetVT.SimpleTy != MVT::f32)
6115 return Register();
6116 if ((Subtarget->isNeonAvailable())) {
6117 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6118 }
6119 return Register();
6120}
6121
6122Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6123 if (RetVT.SimpleTy != MVT::f64)
6124 return Register();
6125 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6126}
6127
6128Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6129 switch (VT.SimpleTy) {
6130 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6131 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6132 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6133 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6134 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6135 default: return Register();
6136 }
6137}
6138
6139// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6140
6141Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6142 if (RetVT.SimpleTy != MVT::f16)
6143 return Register();
6144 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6145 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6146 }
6147 return Register();
6148}
6149
6150Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6151 if (RetVT.SimpleTy != MVT::f16)
6152 return Register();
6153 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6154 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6155 }
6156 return Register();
6157}
6158
6159Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6160 if (RetVT.SimpleTy != MVT::f32)
6161 return Register();
6162 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6163}
6164
6165Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6166 if (RetVT.SimpleTy != MVT::f32)
6167 return Register();
6168 if ((Subtarget->isNeonAvailable())) {
6169 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6170 }
6171 return Register();
6172}
6173
6174Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6175 if (RetVT.SimpleTy != MVT::f64)
6176 return Register();
6177 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6178}
6179
6180Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6181 switch (VT.SimpleTy) {
6182 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6183 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6184 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6185 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6186 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6187 default: return Register();
6188 }
6189}
6190
6191// FastEmit functions for ISD::VECREDUCE_FMIN.
6192
6193Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6194 if (RetVT.SimpleTy != MVT::f16)
6195 return Register();
6196 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6197 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6198 }
6199 return Register();
6200}
6201
6202Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6203 if (RetVT.SimpleTy != MVT::f16)
6204 return Register();
6205 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6206 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6207 }
6208 return Register();
6209}
6210
6211Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6212 if (RetVT.SimpleTy != MVT::f32)
6213 return Register();
6214 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6215}
6216
6217Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6218 if (RetVT.SimpleTy != MVT::f32)
6219 return Register();
6220 if ((Subtarget->isNeonAvailable())) {
6221 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6222 }
6223 return Register();
6224}
6225
6226Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6227 if (RetVT.SimpleTy != MVT::f64)
6228 return Register();
6229 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6230}
6231
6232Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6233 switch (VT.SimpleTy) {
6234 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6235 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6236 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6237 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6238 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6239 default: return Register();
6240 }
6241}
6242
6243// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6244
6245Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6246 if (RetVT.SimpleTy != MVT::f16)
6247 return Register();
6248 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6249 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6250 }
6251 return Register();
6252}
6253
6254Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6255 if (RetVT.SimpleTy != MVT::f16)
6256 return Register();
6257 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6258 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6259 }
6260 return Register();
6261}
6262
6263Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6264 if (RetVT.SimpleTy != MVT::f32)
6265 return Register();
6266 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6267}
6268
6269Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6270 if (RetVT.SimpleTy != MVT::f32)
6271 return Register();
6272 if ((Subtarget->isNeonAvailable())) {
6273 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6274 }
6275 return Register();
6276}
6277
6278Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6279 if (RetVT.SimpleTy != MVT::f64)
6280 return Register();
6281 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6282}
6283
6284Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6285 switch (VT.SimpleTy) {
6286 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6287 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6288 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6289 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6290 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6291 default: return Register();
6292 }
6293}
6294
6295// FastEmit functions for ISD::VECREDUCE_SMAX.
6296
6297Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6298 if (RetVT.SimpleTy != MVT::i8)
6299 return Register();
6300 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6301}
6302
6303Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6304 if (RetVT.SimpleTy != MVT::i8)
6305 return Register();
6306 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6307}
6308
6309Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6310 if (RetVT.SimpleTy != MVT::i16)
6311 return Register();
6312 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6313}
6314
6315Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6316 if (RetVT.SimpleTy != MVT::i16)
6317 return Register();
6318 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6319}
6320
6321Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6322 if (RetVT.SimpleTy != MVT::i32)
6323 return Register();
6324 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6325}
6326
6327Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6328 switch (VT.SimpleTy) {
6329 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6330 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6331 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6332 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6333 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6334 default: return Register();
6335 }
6336}
6337
6338// FastEmit functions for ISD::VECREDUCE_SMIN.
6339
6340Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6341 if (RetVT.SimpleTy != MVT::i8)
6342 return Register();
6343 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6344}
6345
6346Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6347 if (RetVT.SimpleTy != MVT::i8)
6348 return Register();
6349 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6350}
6351
6352Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6353 if (RetVT.SimpleTy != MVT::i16)
6354 return Register();
6355 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6356}
6357
6358Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6359 if (RetVT.SimpleTy != MVT::i16)
6360 return Register();
6361 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6362}
6363
6364Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6365 if (RetVT.SimpleTy != MVT::i32)
6366 return Register();
6367 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6368}
6369
6370Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6371 switch (VT.SimpleTy) {
6372 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6373 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6374 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6375 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6376 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6377 default: return Register();
6378 }
6379}
6380
6381// FastEmit functions for ISD::VECREDUCE_UMAX.
6382
6383Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6384 if (RetVT.SimpleTy != MVT::i8)
6385 return Register();
6386 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6387}
6388
6389Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6390 if (RetVT.SimpleTy != MVT::i8)
6391 return Register();
6392 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6393}
6394
6395Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6396 if (RetVT.SimpleTy != MVT::i16)
6397 return Register();
6398 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6399}
6400
6401Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6402 if (RetVT.SimpleTy != MVT::i16)
6403 return Register();
6404 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6405}
6406
6407Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6408 if (RetVT.SimpleTy != MVT::i32)
6409 return Register();
6410 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6411}
6412
6413Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6414 switch (VT.SimpleTy) {
6415 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6416 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6417 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6418 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6419 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6420 default: return Register();
6421 }
6422}
6423
6424// FastEmit functions for ISD::VECREDUCE_UMIN.
6425
6426Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6427 if (RetVT.SimpleTy != MVT::i8)
6428 return Register();
6429 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6430}
6431
6432Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6433 if (RetVT.SimpleTy != MVT::i8)
6434 return Register();
6435 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6436}
6437
6438Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6439 if (RetVT.SimpleTy != MVT::i16)
6440 return Register();
6441 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6442}
6443
6444Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6445 if (RetVT.SimpleTy != MVT::i16)
6446 return Register();
6447 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6448}
6449
6450Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6451 if (RetVT.SimpleTy != MVT::i32)
6452 return Register();
6453 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6454}
6455
6456Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6457 switch (VT.SimpleTy) {
6458 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6459 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6460 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6461 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6462 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6463 default: return Register();
6464 }
6465}
6466
6467// FastEmit functions for ISD::VECTOR_REVERSE.
6468
6469Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6470 if (RetVT.SimpleTy != MVT::nxv2i1)
6471 return Register();
6472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6473 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6474 }
6475 return Register();
6476}
6477
6478Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6479 if (RetVT.SimpleTy != MVT::nxv4i1)
6480 return Register();
6481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6482 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6483 }
6484 return Register();
6485}
6486
6487Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6488 if (RetVT.SimpleTy != MVT::nxv8i1)
6489 return Register();
6490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6491 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6492 }
6493 return Register();
6494}
6495
6496Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6497 if (RetVT.SimpleTy != MVT::nxv16i1)
6498 return Register();
6499 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6500 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6501 }
6502 return Register();
6503}
6504
6505Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6506 if (RetVT.SimpleTy != MVT::nxv16i8)
6507 return Register();
6508 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6510 }
6511 return Register();
6512}
6513
6514Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6515 if (RetVT.SimpleTy != MVT::nxv8i16)
6516 return Register();
6517 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6519 }
6520 return Register();
6521}
6522
6523Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6524 if (RetVT.SimpleTy != MVT::nxv4i32)
6525 return Register();
6526 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6528 }
6529 return Register();
6530}
6531
6532Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6533 if (RetVT.SimpleTy != MVT::nxv2i64)
6534 return Register();
6535 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6536 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6537 }
6538 return Register();
6539}
6540
6541Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6542 if (RetVT.SimpleTy != MVT::nxv2f16)
6543 return Register();
6544 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6545 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6546 }
6547 return Register();
6548}
6549
6550Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6551 if (RetVT.SimpleTy != MVT::nxv4f16)
6552 return Register();
6553 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6554 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6555 }
6556 return Register();
6557}
6558
6559Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6560 if (RetVT.SimpleTy != MVT::nxv8f16)
6561 return Register();
6562 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6563 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6564 }
6565 return Register();
6566}
6567
6568Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6569 if (RetVT.SimpleTy != MVT::nxv2bf16)
6570 return Register();
6571 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6572 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6573 }
6574 return Register();
6575}
6576
6577Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6578 if (RetVT.SimpleTy != MVT::nxv4bf16)
6579 return Register();
6580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6581 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6582 }
6583 return Register();
6584}
6585
6586Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6587 if (RetVT.SimpleTy != MVT::nxv8bf16)
6588 return Register();
6589 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6590 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6591 }
6592 return Register();
6593}
6594
6595Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6596 if (RetVT.SimpleTy != MVT::nxv2f32)
6597 return Register();
6598 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6599 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6600 }
6601 return Register();
6602}
6603
6604Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6605 if (RetVT.SimpleTy != MVT::nxv4f32)
6606 return Register();
6607 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6608 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6609 }
6610 return Register();
6611}
6612
6613Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6614 if (RetVT.SimpleTy != MVT::nxv2f64)
6615 return Register();
6616 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6617 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6618 }
6619 return Register();
6620}
6621
6622Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6623 switch (VT.SimpleTy) {
6624 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6625 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6626 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6627 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6628 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6629 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6630 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6631 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6632 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6633 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6634 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6635 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6636 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6637 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6638 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6639 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6640 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6641 default: return Register();
6642 }
6643}
6644
6645// Top-level FastEmit function.
6646
6647Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6648 switch (Opcode) {
6649 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6650 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6651 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6652 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6653 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6654 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6655 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6656 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6657 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6658 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6659 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6660 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6661 case AArch64ISD::SQABS: return fastEmit_AArch64ISD_SQABS_r(VT, RetVT, Op0);
6662 case AArch64ISD::SQNEG: return fastEmit_AArch64ISD_SQNEG_r(VT, RetVT, Op0);
6663 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6664 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6665 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6666 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6667 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6668 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6669 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6670 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6671 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6672 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6673 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6674 case ISD::CTLS: return fastEmit_ISD_CTLS_r(VT, RetVT, Op0);
6675 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6676 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6677 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6678 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6679 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6680 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6681 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6682 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6683 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6684 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6685 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6686 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6687 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6688 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6689 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6690 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6691 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6692 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6693 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6694 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6695 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6696 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6697 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6698 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6699 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6700 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6701 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6702 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6703 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6704 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6705 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6706 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6707 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6708 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6709 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6710 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6711 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6712 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6713 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6714 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6715 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6716 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6717 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6718 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6719 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6720 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6721 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6722 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6723 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6724 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6725 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6726 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6727 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6728 default: return Register();
6729 }
6730}
6731
6732// FastEmit functions for AArch64ISD::ADDHN.
6733
6734Register fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6735 if (RetVT.SimpleTy != MVT::v8i8)
6736 return Register();
6737 if ((Subtarget->isNeonAvailable())) {
6738 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv8i16_v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6739 }
6740 return Register();
6741}
6742
6743Register fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6744 if (RetVT.SimpleTy != MVT::v4i16)
6745 return Register();
6746 if ((Subtarget->isNeonAvailable())) {
6747 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv4i32_v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6748 }
6749 return Register();
6750}
6751
6752Register fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6753 if (RetVT.SimpleTy != MVT::v2i32)
6754 return Register();
6755 if ((Subtarget->isNeonAvailable())) {
6756 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDHNv2i64_v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6757 }
6758 return Register();
6759}
6760
6761Register fastEmit_AArch64ISD_ADDHN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6762 switch (VT.SimpleTy) {
6763 case MVT::v8i16: return fastEmit_AArch64ISD_ADDHN_MVT_v8i16_rr(RetVT, Op0, Op1);
6764 case MVT::v4i32: return fastEmit_AArch64ISD_ADDHN_MVT_v4i32_rr(RetVT, Op0, Op1);
6765 case MVT::v2i64: return fastEmit_AArch64ISD_ADDHN_MVT_v2i64_rr(RetVT, Op0, Op1);
6766 default: return Register();
6767 }
6768}
6769
6770// FastEmit functions for AArch64ISD::ADDP.
6771
6772Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6773 if (RetVT.SimpleTy != MVT::v8i8)
6774 return Register();
6775 if ((Subtarget->isNeonAvailable())) {
6776 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6777 }
6778 return Register();
6779}
6780
6781Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6782 if (RetVT.SimpleTy != MVT::v16i8)
6783 return Register();
6784 if ((Subtarget->isNeonAvailable())) {
6785 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6786 }
6787 return Register();
6788}
6789
6790Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6791 if (RetVT.SimpleTy != MVT::v4i16)
6792 return Register();
6793 if ((Subtarget->isNeonAvailable())) {
6794 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6795 }
6796 return Register();
6797}
6798
6799Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6800 if (RetVT.SimpleTy != MVT::v8i16)
6801 return Register();
6802 if ((Subtarget->isNeonAvailable())) {
6803 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6804 }
6805 return Register();
6806}
6807
6808Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6809 if (RetVT.SimpleTy != MVT::v2i32)
6810 return Register();
6811 if ((Subtarget->isNeonAvailable())) {
6812 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6813 }
6814 return Register();
6815}
6816
6817Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6818 if (RetVT.SimpleTy != MVT::v4i32)
6819 return Register();
6820 if ((Subtarget->isNeonAvailable())) {
6821 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6822 }
6823 return Register();
6824}
6825
6826Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6827 if (RetVT.SimpleTy != MVT::v2i64)
6828 return Register();
6829 if ((Subtarget->isNeonAvailable())) {
6830 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6831 }
6832 return Register();
6833}
6834
6835Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6836 if (RetVT.SimpleTy != MVT::v4f16)
6837 return Register();
6838 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6839 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6840 }
6841 return Register();
6842}
6843
6844Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6845 if (RetVT.SimpleTy != MVT::v8f16)
6846 return Register();
6847 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6848 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6849 }
6850 return Register();
6851}
6852
6853Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6854 if (RetVT.SimpleTy != MVT::v2f32)
6855 return Register();
6856 if ((Subtarget->isNeonAvailable())) {
6857 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6858 }
6859 return Register();
6860}
6861
6862Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6863 if (RetVT.SimpleTy != MVT::v4f32)
6864 return Register();
6865 if ((Subtarget->isNeonAvailable())) {
6866 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6867 }
6868 return Register();
6869}
6870
6871Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6872 if (RetVT.SimpleTy != MVT::v2f64)
6873 return Register();
6874 if ((Subtarget->isNeonAvailable())) {
6875 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6876 }
6877 return Register();
6878}
6879
6880Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6881 switch (VT.SimpleTy) {
6882 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6883 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6884 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6885 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6886 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6887 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6888 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6889 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6890 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6891 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6892 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6893 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6894 default: return Register();
6895 }
6896}
6897
6898// FastEmit functions for AArch64ISD::BIC.
6899
6900Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6901 if (RetVT.SimpleTy != MVT::nxv16i8)
6902 return Register();
6903 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6904 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6905 }
6906 return Register();
6907}
6908
6909Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6910 if (RetVT.SimpleTy != MVT::nxv8i16)
6911 return Register();
6912 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6913 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6914 }
6915 return Register();
6916}
6917
6918Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6919 if (RetVT.SimpleTy != MVT::nxv4i32)
6920 return Register();
6921 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6922 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6923 }
6924 return Register();
6925}
6926
6927Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6928 if (RetVT.SimpleTy != MVT::nxv2i64)
6929 return Register();
6930 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6931 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6932 }
6933 return Register();
6934}
6935
6936Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6937 switch (VT.SimpleTy) {
6938 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6939 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6940 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
6941 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
6942 default: return Register();
6943 }
6944}
6945
6946// FastEmit functions for AArch64ISD::CMTST.
6947
6948Register fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6949 if (RetVT.SimpleTy != MVT::v8i8)
6950 return Register();
6951 if ((Subtarget->isNeonAvailable())) {
6952 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6953 }
6954 return Register();
6955}
6956
6957Register fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6958 if (RetVT.SimpleTy != MVT::v16i8)
6959 return Register();
6960 if ((Subtarget->isNeonAvailable())) {
6961 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6962 }
6963 return Register();
6964}
6965
6966Register fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6967 if (RetVT.SimpleTy != MVT::v4i16)
6968 return Register();
6969 if ((Subtarget->isNeonAvailable())) {
6970 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6971 }
6972 return Register();
6973}
6974
6975Register fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6976 if (RetVT.SimpleTy != MVT::v8i16)
6977 return Register();
6978 if ((Subtarget->isNeonAvailable())) {
6979 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6980 }
6981 return Register();
6982}
6983
6984Register fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6985 if (RetVT.SimpleTy != MVT::v2i32)
6986 return Register();
6987 if ((Subtarget->isNeonAvailable())) {
6988 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6989 }
6990 return Register();
6991}
6992
6993Register fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6994 if (RetVT.SimpleTy != MVT::v4i32)
6995 return Register();
6996 if ((Subtarget->isNeonAvailable())) {
6997 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6998 }
6999 return Register();
7000}
7001
7002Register fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7003 if (RetVT.SimpleTy != MVT::v1i64)
7004 return Register();
7005 if ((Subtarget->isNeonAvailable())) {
7006 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7007 }
7008 return Register();
7009}
7010
7011Register fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7012 if (RetVT.SimpleTy != MVT::v2i64)
7013 return Register();
7014 if ((Subtarget->isNeonAvailable())) {
7015 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMTSTv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7016 }
7017 return Register();
7018}
7019
7020Register fastEmit_AArch64ISD_CMTST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7021 switch (VT.SimpleTy) {
7022 case MVT::v8i8: return fastEmit_AArch64ISD_CMTST_MVT_v8i8_rr(RetVT, Op0, Op1);
7023 case MVT::v16i8: return fastEmit_AArch64ISD_CMTST_MVT_v16i8_rr(RetVT, Op0, Op1);
7024 case MVT::v4i16: return fastEmit_AArch64ISD_CMTST_MVT_v4i16_rr(RetVT, Op0, Op1);
7025 case MVT::v8i16: return fastEmit_AArch64ISD_CMTST_MVT_v8i16_rr(RetVT, Op0, Op1);
7026 case MVT::v2i32: return fastEmit_AArch64ISD_CMTST_MVT_v2i32_rr(RetVT, Op0, Op1);
7027 case MVT::v4i32: return fastEmit_AArch64ISD_CMTST_MVT_v4i32_rr(RetVT, Op0, Op1);
7028 case MVT::v1i64: return fastEmit_AArch64ISD_CMTST_MVT_v1i64_rr(RetVT, Op0, Op1);
7029 case MVT::v2i64: return fastEmit_AArch64ISD_CMTST_MVT_v2i64_rr(RetVT, Op0, Op1);
7030 default: return Register();
7031 }
7032}
7033
7034// FastEmit functions for AArch64ISD::CTTZ_ELTS_ZERO_POISON.
7035
7036Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7037 if (RetVT.SimpleTy != MVT::i64)
7038 return Register();
7039 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7040 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_D, RC: &AArch64::GPR64RegClass, Op0, Op1);
7041 }
7042 return Register();
7043}
7044
7045Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7046 if (RetVT.SimpleTy != MVT::i64)
7047 return Register();
7048 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7049 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_S, RC: &AArch64::GPR64RegClass, Op0, Op1);
7050 }
7051 return Register();
7052}
7053
7054Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7055 if (RetVT.SimpleTy != MVT::i64)
7056 return Register();
7057 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7058 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_H, RC: &AArch64::GPR64RegClass, Op0, Op1);
7059 }
7060 return Register();
7061}
7062
7063Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7064 if (RetVT.SimpleTy != MVT::i64)
7065 return Register();
7066 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()))) {
7067 return fastEmitInst_rr(MachineInstOpcode: AArch64::FIRSTP_XPP_B, RC: &AArch64::GPR64RegClass, Op0, Op1);
7068 }
7069 return Register();
7070}
7071
7072Register fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7073 switch (VT.SimpleTy) {
7074 case MVT::nxv2i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv2i1_rr(RetVT, Op0, Op1);
7075 case MVT::nxv4i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv4i1_rr(RetVT, Op0, Op1);
7076 case MVT::nxv8i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv8i1_rr(RetVT, Op0, Op1);
7077 case MVT::nxv16i1: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7078 default: return Register();
7079 }
7080}
7081
7082// FastEmit functions for AArch64ISD::FCMEQ.
7083
7084Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7085 if (RetVT.SimpleTy != MVT::i32)
7086 return Register();
7087 if ((Subtarget->isNeonAvailable())) {
7088 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7089 }
7090 return Register();
7091}
7092
7093Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7094 if (RetVT.SimpleTy != MVT::i64)
7095 return Register();
7096 if ((Subtarget->isNeonAvailable())) {
7097 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7098 }
7099 return Register();
7100}
7101
7102Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7103 if (RetVT.SimpleTy != MVT::v4i16)
7104 return Register();
7105 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7106 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7107 }
7108 return Register();
7109}
7110
7111Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7112 if (RetVT.SimpleTy != MVT::v8i16)
7113 return Register();
7114 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7115 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7116 }
7117 return Register();
7118}
7119
7120Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7121 if (RetVT.SimpleTy != MVT::v2i32)
7122 return Register();
7123 if ((Subtarget->isNeonAvailable())) {
7124 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7125 }
7126 return Register();
7127}
7128
7129Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7130 if (RetVT.SimpleTy != MVT::v4i32)
7131 return Register();
7132 if ((Subtarget->isNeonAvailable())) {
7133 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7134 }
7135 return Register();
7136}
7137
7138Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7139 if (RetVT.SimpleTy != MVT::v1i64)
7140 return Register();
7141 if ((Subtarget->isNeonAvailable())) {
7142 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7143 }
7144 return Register();
7145}
7146
7147Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7148 if (RetVT.SimpleTy != MVT::v2i64)
7149 return Register();
7150 if ((Subtarget->isNeonAvailable())) {
7151 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7152 }
7153 return Register();
7154}
7155
7156Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7157 switch (VT.SimpleTy) {
7158 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7159 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7160 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7161 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7162 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7163 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7164 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7165 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7166 default: return Register();
7167 }
7168}
7169
7170// FastEmit functions for AArch64ISD::FCMGE.
7171
7172Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7173 if (RetVT.SimpleTy != MVT::i32)
7174 return Register();
7175 if ((Subtarget->isNeonAvailable())) {
7176 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7177 }
7178 return Register();
7179}
7180
7181Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7182 if (RetVT.SimpleTy != MVT::i64)
7183 return Register();
7184 if ((Subtarget->isNeonAvailable())) {
7185 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7186 }
7187 return Register();
7188}
7189
7190Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7191 if (RetVT.SimpleTy != MVT::v4i16)
7192 return Register();
7193 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7194 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7195 }
7196 return Register();
7197}
7198
7199Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7200 if (RetVT.SimpleTy != MVT::v8i16)
7201 return Register();
7202 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7203 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7204 }
7205 return Register();
7206}
7207
7208Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7209 if (RetVT.SimpleTy != MVT::v2i32)
7210 return Register();
7211 if ((Subtarget->isNeonAvailable())) {
7212 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7213 }
7214 return Register();
7215}
7216
7217Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7218 if (RetVT.SimpleTy != MVT::v4i32)
7219 return Register();
7220 if ((Subtarget->isNeonAvailable())) {
7221 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7222 }
7223 return Register();
7224}
7225
7226Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7227 if (RetVT.SimpleTy != MVT::v1i64)
7228 return Register();
7229 if ((Subtarget->isNeonAvailable())) {
7230 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7231 }
7232 return Register();
7233}
7234
7235Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7236 if (RetVT.SimpleTy != MVT::v2i64)
7237 return Register();
7238 if ((Subtarget->isNeonAvailable())) {
7239 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7240 }
7241 return Register();
7242}
7243
7244Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7245 switch (VT.SimpleTy) {
7246 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7247 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7248 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7249 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7250 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7251 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7252 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7253 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7254 default: return Register();
7255 }
7256}
7257
7258// FastEmit functions for AArch64ISD::FCMGT.
7259
7260Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7261 if (RetVT.SimpleTy != MVT::i32)
7262 return Register();
7263 if ((Subtarget->isNeonAvailable())) {
7264 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7265 }
7266 return Register();
7267}
7268
7269Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7270 if (RetVT.SimpleTy != MVT::i64)
7271 return Register();
7272 if ((Subtarget->isNeonAvailable())) {
7273 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7274 }
7275 return Register();
7276}
7277
7278Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7279 if (RetVT.SimpleTy != MVT::v4i16)
7280 return Register();
7281 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7282 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7283 }
7284 return Register();
7285}
7286
7287Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7288 if (RetVT.SimpleTy != MVT::v8i16)
7289 return Register();
7290 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7291 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7292 }
7293 return Register();
7294}
7295
7296Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7297 if (RetVT.SimpleTy != MVT::v2i32)
7298 return Register();
7299 if ((Subtarget->isNeonAvailable())) {
7300 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7301 }
7302 return Register();
7303}
7304
7305Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7306 if (RetVT.SimpleTy != MVT::v4i32)
7307 return Register();
7308 if ((Subtarget->isNeonAvailable())) {
7309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7310 }
7311 return Register();
7312}
7313
7314Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7315 if (RetVT.SimpleTy != MVT::v1i64)
7316 return Register();
7317 if ((Subtarget->isNeonAvailable())) {
7318 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7319 }
7320 return Register();
7321}
7322
7323Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7324 if (RetVT.SimpleTy != MVT::v2i64)
7325 return Register();
7326 if ((Subtarget->isNeonAvailable())) {
7327 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7328 }
7329 return Register();
7330}
7331
7332Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7333 switch (VT.SimpleTy) {
7334 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7335 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7336 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7337 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7338 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7339 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7340 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7341 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7342 default: return Register();
7343 }
7344}
7345
7346// FastEmit functions for AArch64ISD::FCMP.
7347
7348Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7349 if (RetVT.SimpleTy != MVT::i32)
7350 return Register();
7351 if ((Subtarget->hasFullFP16())) {
7352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7353 }
7354 return Register();
7355}
7356
7357Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7358 if (RetVT.SimpleTy != MVT::i32)
7359 return Register();
7360 if ((Subtarget->hasFPARMv8())) {
7361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7362 }
7363 return Register();
7364}
7365
7366Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7367 if (RetVT.SimpleTy != MVT::i32)
7368 return Register();
7369 if ((Subtarget->hasFPARMv8())) {
7370 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7371 }
7372 return Register();
7373}
7374
7375Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7376 switch (VT.SimpleTy) {
7377 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7378 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7379 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7380 default: return Register();
7381 }
7382}
7383
7384// FastEmit functions for AArch64ISD::FRECPS.
7385
7386Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7387 if (RetVT.SimpleTy != MVT::f32)
7388 return Register();
7389 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7390}
7391
7392Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7393 if (RetVT.SimpleTy != MVT::f64)
7394 return Register();
7395 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7396}
7397
7398Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7399 if (RetVT.SimpleTy != MVT::v2f32)
7400 return Register();
7401 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7402}
7403
7404Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7405 if (RetVT.SimpleTy != MVT::v4f32)
7406 return Register();
7407 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7408}
7409
7410Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7411 if (RetVT.SimpleTy != MVT::v2f64)
7412 return Register();
7413 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7414}
7415
7416Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7417 if (RetVT.SimpleTy != MVT::nxv8f16)
7418 return Register();
7419 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7420 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7421 }
7422 return Register();
7423}
7424
7425Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7426 if (RetVT.SimpleTy != MVT::nxv4f32)
7427 return Register();
7428 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7429 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7430 }
7431 return Register();
7432}
7433
7434Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7435 if (RetVT.SimpleTy != MVT::nxv2f64)
7436 return Register();
7437 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7438 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7439 }
7440 return Register();
7441}
7442
7443Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7444 switch (VT.SimpleTy) {
7445 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7446 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7447 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7448 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7449 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7450 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7451 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7452 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7453 default: return Register();
7454 }
7455}
7456
7457// FastEmit functions for AArch64ISD::FRSQRTS.
7458
7459Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7460 if (RetVT.SimpleTy != MVT::f32)
7461 return Register();
7462 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7463}
7464
7465Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7466 if (RetVT.SimpleTy != MVT::f64)
7467 return Register();
7468 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7469}
7470
7471Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7472 if (RetVT.SimpleTy != MVT::v2f32)
7473 return Register();
7474 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7475}
7476
7477Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7478 if (RetVT.SimpleTy != MVT::v4f32)
7479 return Register();
7480 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7481}
7482
7483Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7484 if (RetVT.SimpleTy != MVT::v2f64)
7485 return Register();
7486 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7487}
7488
7489Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7490 if (RetVT.SimpleTy != MVT::nxv8f16)
7491 return Register();
7492 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7493 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7494 }
7495 return Register();
7496}
7497
7498Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7499 if (RetVT.SimpleTy != MVT::nxv4f32)
7500 return Register();
7501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7502 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7503 }
7504 return Register();
7505}
7506
7507Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7508 if (RetVT.SimpleTy != MVT::nxv2f64)
7509 return Register();
7510 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7511 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7512 }
7513 return Register();
7514}
7515
7516Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7517 switch (VT.SimpleTy) {
7518 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7519 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7520 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7521 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7522 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7523 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7524 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7525 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7526 default: return Register();
7527 }
7528}
7529
7530// FastEmit functions for AArch64ISD::PMULL.
7531
7532Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7533 if (RetVT.SimpleTy != MVT::v8i16)
7534 return Register();
7535 if ((Subtarget->isNeonAvailable())) {
7536 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7537 }
7538 return Register();
7539}
7540
7541Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7542 if (RetVT.SimpleTy != MVT::v16i8)
7543 return Register();
7544 if ((Subtarget->hasAES())) {
7545 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7546 }
7547 return Register();
7548}
7549
7550Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7551 switch (VT.SimpleTy) {
7552 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7553 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7554 default: return Register();
7555 }
7556}
7557
7558// FastEmit functions for AArch64ISD::PTEST.
7559
7560Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7561 if (RetVT.SimpleTy != MVT::i32)
7562 return Register();
7563 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7564 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7565 }
7566 return Register();
7567}
7568
7569Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7570 switch (VT.SimpleTy) {
7571 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7572 default: return Register();
7573 }
7574}
7575
7576// FastEmit functions for AArch64ISD::PTEST_ANY.
7577
7578Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7579 if (RetVT.SimpleTy != MVT::i32)
7580 return Register();
7581 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7582 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7583 }
7584 return Register();
7585}
7586
7587Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7588 switch (VT.SimpleTy) {
7589 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7590 default: return Register();
7591 }
7592}
7593
7594// FastEmit functions for AArch64ISD::PTEST_FIRST.
7595
7596Register fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7597 if (RetVT.SimpleTy != MVT::i32)
7598 return Register();
7599 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7600 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_FIRST, RC: &AArch64::PPRRegClass, Op0, Op1);
7601 }
7602 return Register();
7603}
7604
7605Register fastEmit_AArch64ISD_PTEST_FIRST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7606 switch (VT.SimpleTy) {
7607 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_FIRST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7608 default: return Register();
7609 }
7610}
7611
7612// FastEmit functions for AArch64ISD::SMULL.
7613
7614Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7615 if (RetVT.SimpleTy != MVT::v8i16)
7616 return Register();
7617 if ((Subtarget->isNeonAvailable())) {
7618 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7619 }
7620 return Register();
7621}
7622
7623Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7624 if (RetVT.SimpleTy != MVT::v4i32)
7625 return Register();
7626 if ((Subtarget->isNeonAvailable())) {
7627 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7628 }
7629 return Register();
7630}
7631
7632Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7633 if (RetVT.SimpleTy != MVT::v2i64)
7634 return Register();
7635 if ((Subtarget->isNeonAvailable())) {
7636 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7637 }
7638 return Register();
7639}
7640
7641Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7642 switch (VT.SimpleTy) {
7643 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7644 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7645 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7646 default: return Register();
7647 }
7648}
7649
7650// FastEmit functions for AArch64ISD::SQADD.
7651
7652Register fastEmit_AArch64ISD_SQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7653 if (RetVT.SimpleTy != MVT::f32)
7654 return Register();
7655 if ((Subtarget->isNeonAvailable())) {
7656 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7657 }
7658 return Register();
7659}
7660
7661Register fastEmit_AArch64ISD_SQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7662 if (RetVT.SimpleTy != MVT::f64)
7663 return Register();
7664 if ((Subtarget->isNeonAvailable())) {
7665 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7666 }
7667 return Register();
7668}
7669
7670Register fastEmit_AArch64ISD_SQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7671 switch (VT.SimpleTy) {
7672 case MVT::f32: return fastEmit_AArch64ISD_SQADD_MVT_f32_rr(RetVT, Op0, Op1);
7673 case MVT::f64: return fastEmit_AArch64ISD_SQADD_MVT_f64_rr(RetVT, Op0, Op1);
7674 default: return Register();
7675 }
7676}
7677
7678// FastEmit functions for AArch64ISD::SQDMULH.
7679
7680Register fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7681 if (RetVT.SimpleTy != MVT::f32)
7682 return Register();
7683 if ((Subtarget->isNeonAvailable())) {
7684 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7685 }
7686 return Register();
7687}
7688
7689Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7690 if (RetVT.SimpleTy != MVT::v4i16)
7691 return Register();
7692 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7693}
7694
7695Register fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7696 if (RetVT.SimpleTy != MVT::v8i16)
7697 return Register();
7698 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7699}
7700
7701Register fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7702 if (RetVT.SimpleTy != MVT::v2i32)
7703 return Register();
7704 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7705}
7706
7707Register fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7708 if (RetVT.SimpleTy != MVT::v4i32)
7709 return Register();
7710 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULHv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7711}
7712
7713Register fastEmit_AArch64ISD_SQDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7714 switch (VT.SimpleTy) {
7715 case MVT::f32: return fastEmit_AArch64ISD_SQDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7716 case MVT::v4i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i16_rr(RetVT, Op0, Op1);
7717 case MVT::v8i16: return fastEmit_AArch64ISD_SQDMULH_MVT_v8i16_rr(RetVT, Op0, Op1);
7718 case MVT::v2i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v2i32_rr(RetVT, Op0, Op1);
7719 case MVT::v4i32: return fastEmit_AArch64ISD_SQDMULH_MVT_v4i32_rr(RetVT, Op0, Op1);
7720 default: return Register();
7721 }
7722}
7723
7724// FastEmit functions for AArch64ISD::SQDMULL.
7725
7726Register fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7727 if (RetVT.SimpleTy != MVT::f64)
7728 return Register();
7729 if ((Subtarget->isNeonAvailable())) {
7730 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQDMULLi32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7731 }
7732 return Register();
7733}
7734
7735Register fastEmit_AArch64ISD_SQDMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7736 switch (VT.SimpleTy) {
7737 case MVT::f32: return fastEmit_AArch64ISD_SQDMULL_MVT_f32_rr(RetVT, Op0, Op1);
7738 default: return Register();
7739 }
7740}
7741
7742// FastEmit functions for AArch64ISD::SQRDMULH.
7743
7744Register fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7745 if (RetVT.SimpleTy != MVT::f32)
7746 return Register();
7747 if ((Subtarget->isNeonAvailable())) {
7748 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRDMULHv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7749 }
7750 return Register();
7751}
7752
7753Register fastEmit_AArch64ISD_SQRDMULH_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7754 switch (VT.SimpleTy) {
7755 case MVT::f32: return fastEmit_AArch64ISD_SQRDMULH_MVT_f32_rr(RetVT, Op0, Op1);
7756 default: return Register();
7757 }
7758}
7759
7760// FastEmit functions for AArch64ISD::SQRSHL.
7761
7762Register fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7763 if (RetVT.SimpleTy != MVT::f32)
7764 return Register();
7765 if ((Subtarget->isNeonAvailable())) {
7766 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7767 }
7768 return Register();
7769}
7770
7771Register fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7772 if (RetVT.SimpleTy != MVT::f64)
7773 return Register();
7774 if ((Subtarget->isNeonAvailable())) {
7775 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7776 }
7777 return Register();
7778}
7779
7780Register fastEmit_AArch64ISD_SQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7781 switch (VT.SimpleTy) {
7782 case MVT::f32: return fastEmit_AArch64ISD_SQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
7783 case MVT::f64: return fastEmit_AArch64ISD_SQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
7784 default: return Register();
7785 }
7786}
7787
7788// FastEmit functions for AArch64ISD::SQSHL.
7789
7790Register fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7791 if (RetVT.SimpleTy != MVT::f32)
7792 return Register();
7793 if ((Subtarget->isNeonAvailable())) {
7794 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7795 }
7796 return Register();
7797}
7798
7799Register fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7800 if (RetVT.SimpleTy != MVT::f64)
7801 return Register();
7802 if ((Subtarget->isNeonAvailable())) {
7803 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7804 }
7805 return Register();
7806}
7807
7808Register fastEmit_AArch64ISD_SQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7809 switch (VT.SimpleTy) {
7810 case MVT::f32: return fastEmit_AArch64ISD_SQSHL_MVT_f32_rr(RetVT, Op0, Op1);
7811 case MVT::f64: return fastEmit_AArch64ISD_SQSHL_MVT_f64_rr(RetVT, Op0, Op1);
7812 default: return Register();
7813 }
7814}
7815
7816// FastEmit functions for AArch64ISD::SQSUB.
7817
7818Register fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7819 if (RetVT.SimpleTy != MVT::f32)
7820 return Register();
7821 if ((Subtarget->isNeonAvailable())) {
7822 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7823 }
7824 return Register();
7825}
7826
7827Register fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7828 if (RetVT.SimpleTy != MVT::f64)
7829 return Register();
7830 if ((Subtarget->isNeonAvailable())) {
7831 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7832 }
7833 return Register();
7834}
7835
7836Register fastEmit_AArch64ISD_SQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7837 switch (VT.SimpleTy) {
7838 case MVT::f32: return fastEmit_AArch64ISD_SQSUB_MVT_f32_rr(RetVT, Op0, Op1);
7839 case MVT::f64: return fastEmit_AArch64ISD_SQSUB_MVT_f64_rr(RetVT, Op0, Op1);
7840 default: return Register();
7841 }
7842}
7843
7844// FastEmit functions for AArch64ISD::STRICT_FCMP.
7845
7846Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7847 if (RetVT.SimpleTy != MVT::i32)
7848 return Register();
7849 if ((Subtarget->hasFullFP16())) {
7850 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7851 }
7852 return Register();
7853}
7854
7855Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7856 if (RetVT.SimpleTy != MVT::i32)
7857 return Register();
7858 if ((Subtarget->hasFPARMv8())) {
7859 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7860 }
7861 return Register();
7862}
7863
7864Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7865 if (RetVT.SimpleTy != MVT::i32)
7866 return Register();
7867 if ((Subtarget->hasFPARMv8())) {
7868 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7869 }
7870 return Register();
7871}
7872
7873Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7874 switch (VT.SimpleTy) {
7875 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7876 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7877 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7878 default: return Register();
7879 }
7880}
7881
7882// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7883
7884Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7885 if (RetVT.SimpleTy != MVT::i32)
7886 return Register();
7887 if ((Subtarget->hasFullFP16())) {
7888 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7889 }
7890 return Register();
7891}
7892
7893Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7894 if (RetVT.SimpleTy != MVT::i32)
7895 return Register();
7896 if ((Subtarget->hasFPARMv8())) {
7897 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7898 }
7899 return Register();
7900}
7901
7902Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7903 if (RetVT.SimpleTy != MVT::i32)
7904 return Register();
7905 if ((Subtarget->hasFPARMv8())) {
7906 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7907 }
7908 return Register();
7909}
7910
7911Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7912 switch (VT.SimpleTy) {
7913 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
7914 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
7915 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
7916 default: return Register();
7917 }
7918}
7919
7920// FastEmit functions for AArch64ISD::SUQADD.
7921
7922Register fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7923 if (RetVT.SimpleTy != MVT::f32)
7924 return Register();
7925 if ((Subtarget->isNeonAvailable())) {
7926 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7927 }
7928 return Register();
7929}
7930
7931Register fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7932 if (RetVT.SimpleTy != MVT::f64)
7933 return Register();
7934 if ((Subtarget->isNeonAvailable())) {
7935 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7936 }
7937 return Register();
7938}
7939
7940Register fastEmit_AArch64ISD_SUQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7941 switch (VT.SimpleTy) {
7942 case MVT::f32: return fastEmit_AArch64ISD_SUQADD_MVT_f32_rr(RetVT, Op0, Op1);
7943 case MVT::f64: return fastEmit_AArch64ISD_SUQADD_MVT_f64_rr(RetVT, Op0, Op1);
7944 default: return Register();
7945 }
7946}
7947
7948// FastEmit functions for AArch64ISD::TBL.
7949
7950Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7951 if (RetVT.SimpleTy != MVT::nxv16i8)
7952 return Register();
7953 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7954 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7955 }
7956 return Register();
7957}
7958
7959Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7960 if (RetVT.SimpleTy != MVT::nxv8i16)
7961 return Register();
7962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7963 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7964 }
7965 return Register();
7966}
7967
7968Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7969 if (RetVT.SimpleTy != MVT::nxv4i32)
7970 return Register();
7971 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7972 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7973 }
7974 return Register();
7975}
7976
7977Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7978 if (RetVT.SimpleTy != MVT::nxv2i64)
7979 return Register();
7980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7981 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7982 }
7983 return Register();
7984}
7985
7986Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7987 switch (VT.SimpleTy) {
7988 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7989 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7990 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7991 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7992 default: return Register();
7993 }
7994}
7995
7996// FastEmit functions for AArch64ISD::TRN1.
7997
7998Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7999 if (RetVT.SimpleTy != MVT::v8i8)
8000 return Register();
8001 if ((Subtarget->isNeonAvailable())) {
8002 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8003 }
8004 return Register();
8005}
8006
8007Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8008 if (RetVT.SimpleTy != MVT::v16i8)
8009 return Register();
8010 if ((Subtarget->isNeonAvailable())) {
8011 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8012 }
8013 return Register();
8014}
8015
8016Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8017 if (RetVT.SimpleTy != MVT::v4i16)
8018 return Register();
8019 if ((Subtarget->isNeonAvailable())) {
8020 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8021 }
8022 return Register();
8023}
8024
8025Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8026 if (RetVT.SimpleTy != MVT::v8i16)
8027 return Register();
8028 if ((Subtarget->isNeonAvailable())) {
8029 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8030 }
8031 return Register();
8032}
8033
8034Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8035 if (RetVT.SimpleTy != MVT::v2i32)
8036 return Register();
8037 if ((Subtarget->isNeonAvailable())) {
8038 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8039 }
8040 return Register();
8041}
8042
8043Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8044 if (RetVT.SimpleTy != MVT::v4i32)
8045 return Register();
8046 if ((Subtarget->isNeonAvailable())) {
8047 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8048 }
8049 return Register();
8050}
8051
8052Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8053 if (RetVT.SimpleTy != MVT::v2i64)
8054 return Register();
8055 if ((Subtarget->isNeonAvailable())) {
8056 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8057 }
8058 return Register();
8059}
8060
8061Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8062 if (RetVT.SimpleTy != MVT::v4f16)
8063 return Register();
8064 if ((Subtarget->isNeonAvailable())) {
8065 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8066 }
8067 return Register();
8068}
8069
8070Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8071 if (RetVT.SimpleTy != MVT::v8f16)
8072 return Register();
8073 if ((Subtarget->isNeonAvailable())) {
8074 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8075 }
8076 return Register();
8077}
8078
8079Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8080 if (RetVT.SimpleTy != MVT::v4bf16)
8081 return Register();
8082 if ((Subtarget->isNeonAvailable())) {
8083 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8084 }
8085 return Register();
8086}
8087
8088Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8089 if (RetVT.SimpleTy != MVT::v8bf16)
8090 return Register();
8091 if ((Subtarget->isNeonAvailable())) {
8092 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8093 }
8094 return Register();
8095}
8096
8097Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8098 if (RetVT.SimpleTy != MVT::v2f32)
8099 return Register();
8100 if ((Subtarget->isNeonAvailable())) {
8101 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8102 }
8103 return Register();
8104}
8105
8106Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8107 if (RetVT.SimpleTy != MVT::v4f32)
8108 return Register();
8109 if ((Subtarget->isNeonAvailable())) {
8110 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8111 }
8112 return Register();
8113}
8114
8115Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8116 if (RetVT.SimpleTy != MVT::v2f64)
8117 return Register();
8118 if ((Subtarget->isNeonAvailable())) {
8119 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8120 }
8121 return Register();
8122}
8123
8124Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8125 if (RetVT.SimpleTy != MVT::nxv2i1)
8126 return Register();
8127 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8128 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8129 }
8130 return Register();
8131}
8132
8133Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8134 if (RetVT.SimpleTy != MVT::nxv4i1)
8135 return Register();
8136 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8137 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8138 }
8139 return Register();
8140}
8141
8142Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8143 if (RetVT.SimpleTy != MVT::nxv8i1)
8144 return Register();
8145 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8146 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8147 }
8148 return Register();
8149}
8150
8151Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8152 if (RetVT.SimpleTy != MVT::nxv16i1)
8153 return Register();
8154 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8155 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8156 }
8157 return Register();
8158}
8159
8160Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8161 if (RetVT.SimpleTy != MVT::nxv16i8)
8162 return Register();
8163 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8164 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8165 }
8166 return Register();
8167}
8168
8169Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8170 if (RetVT.SimpleTy != MVT::nxv8i16)
8171 return Register();
8172 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8173 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8174 }
8175 return Register();
8176}
8177
8178Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8179 if (RetVT.SimpleTy != MVT::nxv4i32)
8180 return Register();
8181 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8182 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8183 }
8184 return Register();
8185}
8186
8187Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8188 if (RetVT.SimpleTy != MVT::nxv2i64)
8189 return Register();
8190 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8191 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8192 }
8193 return Register();
8194}
8195
8196Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8197 if (RetVT.SimpleTy != MVT::nxv2f16)
8198 return Register();
8199 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8200 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8201 }
8202 return Register();
8203}
8204
8205Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8206 if (RetVT.SimpleTy != MVT::nxv4f16)
8207 return Register();
8208 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8209 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8210 }
8211 return Register();
8212}
8213
8214Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8215 if (RetVT.SimpleTy != MVT::nxv8f16)
8216 return Register();
8217 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8218 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8219 }
8220 return Register();
8221}
8222
8223Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8224 if (RetVT.SimpleTy != MVT::nxv2bf16)
8225 return Register();
8226 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8227 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8228 }
8229 return Register();
8230}
8231
8232Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8233 if (RetVT.SimpleTy != MVT::nxv4bf16)
8234 return Register();
8235 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8236 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8237 }
8238 return Register();
8239}
8240
8241Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8242 if (RetVT.SimpleTy != MVT::nxv8bf16)
8243 return Register();
8244 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8245 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8246 }
8247 return Register();
8248}
8249
8250Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8251 if (RetVT.SimpleTy != MVT::nxv2f32)
8252 return Register();
8253 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8254 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8255 }
8256 return Register();
8257}
8258
8259Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8260 if (RetVT.SimpleTy != MVT::nxv4f32)
8261 return Register();
8262 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8263 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8264 }
8265 return Register();
8266}
8267
8268Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8269 if (RetVT.SimpleTy != MVT::nxv2f64)
8270 return Register();
8271 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8272 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8273 }
8274 return Register();
8275}
8276
8277Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8278 switch (VT.SimpleTy) {
8279 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8280 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8281 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8282 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8283 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8284 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8285 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8286 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8287 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8288 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8289 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8290 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8291 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8292 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8293 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8294 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8295 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8296 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8297 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8298 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8299 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8300 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8301 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8302 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8303 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8304 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8305 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8306 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8307 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8308 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8309 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8310 default: return Register();
8311 }
8312}
8313
8314// FastEmit functions for AArch64ISD::TRN2.
8315
8316Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8317 if (RetVT.SimpleTy != MVT::v8i8)
8318 return Register();
8319 if ((Subtarget->isNeonAvailable())) {
8320 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8321 }
8322 return Register();
8323}
8324
8325Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8326 if (RetVT.SimpleTy != MVT::v16i8)
8327 return Register();
8328 if ((Subtarget->isNeonAvailable())) {
8329 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8330 }
8331 return Register();
8332}
8333
8334Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8335 if (RetVT.SimpleTy != MVT::v4i16)
8336 return Register();
8337 if ((Subtarget->isNeonAvailable())) {
8338 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8339 }
8340 return Register();
8341}
8342
8343Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8344 if (RetVT.SimpleTy != MVT::v8i16)
8345 return Register();
8346 if ((Subtarget->isNeonAvailable())) {
8347 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8348 }
8349 return Register();
8350}
8351
8352Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8353 if (RetVT.SimpleTy != MVT::v2i32)
8354 return Register();
8355 if ((Subtarget->isNeonAvailable())) {
8356 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8357 }
8358 return Register();
8359}
8360
8361Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8362 if (RetVT.SimpleTy != MVT::v4i32)
8363 return Register();
8364 if ((Subtarget->isNeonAvailable())) {
8365 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8366 }
8367 return Register();
8368}
8369
8370Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8371 if (RetVT.SimpleTy != MVT::v2i64)
8372 return Register();
8373 if ((Subtarget->isNeonAvailable())) {
8374 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8375 }
8376 return Register();
8377}
8378
8379Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8380 if (RetVT.SimpleTy != MVT::v4f16)
8381 return Register();
8382 if ((Subtarget->isNeonAvailable())) {
8383 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8384 }
8385 return Register();
8386}
8387
8388Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8389 if (RetVT.SimpleTy != MVT::v8f16)
8390 return Register();
8391 if ((Subtarget->isNeonAvailable())) {
8392 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8393 }
8394 return Register();
8395}
8396
8397Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8398 if (RetVT.SimpleTy != MVT::v4bf16)
8399 return Register();
8400 if ((Subtarget->isNeonAvailable())) {
8401 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8402 }
8403 return Register();
8404}
8405
8406Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8407 if (RetVT.SimpleTy != MVT::v8bf16)
8408 return Register();
8409 if ((Subtarget->isNeonAvailable())) {
8410 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8411 }
8412 return Register();
8413}
8414
8415Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8416 if (RetVT.SimpleTy != MVT::v2f32)
8417 return Register();
8418 if ((Subtarget->isNeonAvailable())) {
8419 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8420 }
8421 return Register();
8422}
8423
8424Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8425 if (RetVT.SimpleTy != MVT::v4f32)
8426 return Register();
8427 if ((Subtarget->isNeonAvailable())) {
8428 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8429 }
8430 return Register();
8431}
8432
8433Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8434 if (RetVT.SimpleTy != MVT::v2f64)
8435 return Register();
8436 if ((Subtarget->isNeonAvailable())) {
8437 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8438 }
8439 return Register();
8440}
8441
8442Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8443 if (RetVT.SimpleTy != MVT::nxv2i1)
8444 return Register();
8445 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8446 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8447 }
8448 return Register();
8449}
8450
8451Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8452 if (RetVT.SimpleTy != MVT::nxv4i1)
8453 return Register();
8454 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8455 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8456 }
8457 return Register();
8458}
8459
8460Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8461 if (RetVT.SimpleTy != MVT::nxv8i1)
8462 return Register();
8463 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8464 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8465 }
8466 return Register();
8467}
8468
8469Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8470 if (RetVT.SimpleTy != MVT::nxv16i1)
8471 return Register();
8472 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8473 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8474 }
8475 return Register();
8476}
8477
8478Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8479 if (RetVT.SimpleTy != MVT::nxv16i8)
8480 return Register();
8481 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8482 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8483 }
8484 return Register();
8485}
8486
8487Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8488 if (RetVT.SimpleTy != MVT::nxv8i16)
8489 return Register();
8490 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8491 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8492 }
8493 return Register();
8494}
8495
8496Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8497 if (RetVT.SimpleTy != MVT::nxv4i32)
8498 return Register();
8499 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8500 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8501 }
8502 return Register();
8503}
8504
8505Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8506 if (RetVT.SimpleTy != MVT::nxv2i64)
8507 return Register();
8508 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8509 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8510 }
8511 return Register();
8512}
8513
8514Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8515 if (RetVT.SimpleTy != MVT::nxv2f16)
8516 return Register();
8517 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8518 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8519 }
8520 return Register();
8521}
8522
8523Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8524 if (RetVT.SimpleTy != MVT::nxv4f16)
8525 return Register();
8526 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8527 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8528 }
8529 return Register();
8530}
8531
8532Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8533 if (RetVT.SimpleTy != MVT::nxv8f16)
8534 return Register();
8535 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8536 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8537 }
8538 return Register();
8539}
8540
8541Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8542 if (RetVT.SimpleTy != MVT::nxv2bf16)
8543 return Register();
8544 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8545 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8546 }
8547 return Register();
8548}
8549
8550Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8551 if (RetVT.SimpleTy != MVT::nxv4bf16)
8552 return Register();
8553 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8554 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8555 }
8556 return Register();
8557}
8558
8559Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8560 if (RetVT.SimpleTy != MVT::nxv8bf16)
8561 return Register();
8562 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8563 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8564 }
8565 return Register();
8566}
8567
8568Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8569 if (RetVT.SimpleTy != MVT::nxv2f32)
8570 return Register();
8571 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8572 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8573 }
8574 return Register();
8575}
8576
8577Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8578 if (RetVT.SimpleTy != MVT::nxv4f32)
8579 return Register();
8580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8581 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8582 }
8583 return Register();
8584}
8585
8586Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8587 if (RetVT.SimpleTy != MVT::nxv2f64)
8588 return Register();
8589 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8590 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8591 }
8592 return Register();
8593}
8594
8595Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8596 switch (VT.SimpleTy) {
8597 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8598 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8599 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8600 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8601 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8602 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8603 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8604 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8605 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8606 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8607 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8608 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8609 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8610 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8611 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8612 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8613 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8614 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8615 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8616 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8617 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8618 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8619 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8620 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8621 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8622 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8623 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8624 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8625 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8626 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8627 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8628 default: return Register();
8629 }
8630}
8631
8632// FastEmit functions for AArch64ISD::UMULL.
8633
8634Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8635 if (RetVT.SimpleTy != MVT::v8i16)
8636 return Register();
8637 if ((Subtarget->isNeonAvailable())) {
8638 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8639 }
8640 return Register();
8641}
8642
8643Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8644 if (RetVT.SimpleTy != MVT::v4i32)
8645 return Register();
8646 if ((Subtarget->isNeonAvailable())) {
8647 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8648 }
8649 return Register();
8650}
8651
8652Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8653 if (RetVT.SimpleTy != MVT::v2i64)
8654 return Register();
8655 if ((Subtarget->isNeonAvailable())) {
8656 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8657 }
8658 return Register();
8659}
8660
8661Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8662 switch (VT.SimpleTy) {
8663 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8664 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8665 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8666 default: return Register();
8667 }
8668}
8669
8670// FastEmit functions for AArch64ISD::UQADD.
8671
8672Register fastEmit_AArch64ISD_UQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8673 if (RetVT.SimpleTy != MVT::f32)
8674 return Register();
8675 if ((Subtarget->isNeonAvailable())) {
8676 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8677 }
8678 return Register();
8679}
8680
8681Register fastEmit_AArch64ISD_UQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8682 if (RetVT.SimpleTy != MVT::f64)
8683 return Register();
8684 if ((Subtarget->isNeonAvailable())) {
8685 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8686 }
8687 return Register();
8688}
8689
8690Register fastEmit_AArch64ISD_UQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8691 switch (VT.SimpleTy) {
8692 case MVT::f32: return fastEmit_AArch64ISD_UQADD_MVT_f32_rr(RetVT, Op0, Op1);
8693 case MVT::f64: return fastEmit_AArch64ISD_UQADD_MVT_f64_rr(RetVT, Op0, Op1);
8694 default: return Register();
8695 }
8696}
8697
8698// FastEmit functions for AArch64ISD::UQRSHL.
8699
8700Register fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8701 if (RetVT.SimpleTy != MVT::f32)
8702 return Register();
8703 if ((Subtarget->isNeonAvailable())) {
8704 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8705 }
8706 return Register();
8707}
8708
8709Register fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8710 if (RetVT.SimpleTy != MVT::f64)
8711 return Register();
8712 if ((Subtarget->isNeonAvailable())) {
8713 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQRSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8714 }
8715 return Register();
8716}
8717
8718Register fastEmit_AArch64ISD_UQRSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8719 switch (VT.SimpleTy) {
8720 case MVT::f32: return fastEmit_AArch64ISD_UQRSHL_MVT_f32_rr(RetVT, Op0, Op1);
8721 case MVT::f64: return fastEmit_AArch64ISD_UQRSHL_MVT_f64_rr(RetVT, Op0, Op1);
8722 default: return Register();
8723 }
8724}
8725
8726// FastEmit functions for AArch64ISD::UQSHL.
8727
8728Register fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8729 if (RetVT.SimpleTy != MVT::f32)
8730 return Register();
8731 if ((Subtarget->isNeonAvailable())) {
8732 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8733 }
8734 return Register();
8735}
8736
8737Register fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8738 if (RetVT.SimpleTy != MVT::f64)
8739 return Register();
8740 if ((Subtarget->isNeonAvailable())) {
8741 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSHLv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8742 }
8743 return Register();
8744}
8745
8746Register fastEmit_AArch64ISD_UQSHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8747 switch (VT.SimpleTy) {
8748 case MVT::f32: return fastEmit_AArch64ISD_UQSHL_MVT_f32_rr(RetVT, Op0, Op1);
8749 case MVT::f64: return fastEmit_AArch64ISD_UQSHL_MVT_f64_rr(RetVT, Op0, Op1);
8750 default: return Register();
8751 }
8752}
8753
8754// FastEmit functions for AArch64ISD::UQSUB.
8755
8756Register fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8757 if (RetVT.SimpleTy != MVT::f32)
8758 return Register();
8759 if ((Subtarget->isNeonAvailable())) {
8760 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8761 }
8762 return Register();
8763}
8764
8765Register fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8766 if (RetVT.SimpleTy != MVT::f64)
8767 return Register();
8768 if ((Subtarget->isNeonAvailable())) {
8769 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8770 }
8771 return Register();
8772}
8773
8774Register fastEmit_AArch64ISD_UQSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8775 switch (VT.SimpleTy) {
8776 case MVT::f32: return fastEmit_AArch64ISD_UQSUB_MVT_f32_rr(RetVT, Op0, Op1);
8777 case MVT::f64: return fastEmit_AArch64ISD_UQSUB_MVT_f64_rr(RetVT, Op0, Op1);
8778 default: return Register();
8779 }
8780}
8781
8782// FastEmit functions for AArch64ISD::USQADD.
8783
8784Register fastEmit_AArch64ISD_USQADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
8785 if (RetVT.SimpleTy != MVT::f32)
8786 return Register();
8787 if ((Subtarget->isNeonAvailable())) {
8788 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8789 }
8790 return Register();
8791}
8792
8793Register fastEmit_AArch64ISD_USQADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
8794 if (RetVT.SimpleTy != MVT::f64)
8795 return Register();
8796 if ((Subtarget->isNeonAvailable())) {
8797 return fastEmitInst_rr(MachineInstOpcode: AArch64::USQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8798 }
8799 return Register();
8800}
8801
8802Register fastEmit_AArch64ISD_USQADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8803 switch (VT.SimpleTy) {
8804 case MVT::f32: return fastEmit_AArch64ISD_USQADD_MVT_f32_rr(RetVT, Op0, Op1);
8805 case MVT::f64: return fastEmit_AArch64ISD_USQADD_MVT_f64_rr(RetVT, Op0, Op1);
8806 default: return Register();
8807 }
8808}
8809
8810// FastEmit functions for AArch64ISD::UZP1.
8811
8812Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8813 if (RetVT.SimpleTy != MVT::v8i8)
8814 return Register();
8815 if ((Subtarget->isNeonAvailable())) {
8816 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8817 }
8818 return Register();
8819}
8820
8821Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8822 if (RetVT.SimpleTy != MVT::v16i8)
8823 return Register();
8824 if ((Subtarget->isNeonAvailable())) {
8825 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8826 }
8827 return Register();
8828}
8829
8830Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8831 if (RetVT.SimpleTy != MVT::v4i16)
8832 return Register();
8833 if ((Subtarget->isNeonAvailable())) {
8834 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8835 }
8836 return Register();
8837}
8838
8839Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8840 if (RetVT.SimpleTy != MVT::v8i16)
8841 return Register();
8842 if ((Subtarget->isNeonAvailable())) {
8843 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8844 }
8845 return Register();
8846}
8847
8848Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8849 if (RetVT.SimpleTy != MVT::v2i32)
8850 return Register();
8851 if ((Subtarget->isNeonAvailable())) {
8852 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8853 }
8854 return Register();
8855}
8856
8857Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8858 if (RetVT.SimpleTy != MVT::v4i32)
8859 return Register();
8860 if ((Subtarget->isNeonAvailable())) {
8861 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8862 }
8863 return Register();
8864}
8865
8866Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8867 if (RetVT.SimpleTy != MVT::v2i64)
8868 return Register();
8869 if ((Subtarget->isNeonAvailable())) {
8870 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8871 }
8872 return Register();
8873}
8874
8875Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8876 if (RetVT.SimpleTy != MVT::v4f16)
8877 return Register();
8878 if ((Subtarget->isNeonAvailable())) {
8879 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8880 }
8881 return Register();
8882}
8883
8884Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8885 if (RetVT.SimpleTy != MVT::v8f16)
8886 return Register();
8887 if ((Subtarget->isNeonAvailable())) {
8888 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8889 }
8890 return Register();
8891}
8892
8893Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8894 if (RetVT.SimpleTy != MVT::v4bf16)
8895 return Register();
8896 if ((Subtarget->isNeonAvailable())) {
8897 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8898 }
8899 return Register();
8900}
8901
8902Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8903 if (RetVT.SimpleTy != MVT::v8bf16)
8904 return Register();
8905 if ((Subtarget->isNeonAvailable())) {
8906 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8907 }
8908 return Register();
8909}
8910
8911Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8912 if (RetVT.SimpleTy != MVT::v2f32)
8913 return Register();
8914 if ((Subtarget->isNeonAvailable())) {
8915 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8916 }
8917 return Register();
8918}
8919
8920Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8921 if (RetVT.SimpleTy != MVT::v4f32)
8922 return Register();
8923 if ((Subtarget->isNeonAvailable())) {
8924 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8925 }
8926 return Register();
8927}
8928
8929Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8930 if (RetVT.SimpleTy != MVT::v2f64)
8931 return Register();
8932 if ((Subtarget->isNeonAvailable())) {
8933 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8934 }
8935 return Register();
8936}
8937
8938Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8939 if (RetVT.SimpleTy != MVT::nxv2i1)
8940 return Register();
8941 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8942 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8943 }
8944 return Register();
8945}
8946
8947Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8948 if (RetVT.SimpleTy != MVT::nxv4i1)
8949 return Register();
8950 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8951 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8952 }
8953 return Register();
8954}
8955
8956Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8957 if (RetVT.SimpleTy != MVT::nxv8i1)
8958 return Register();
8959 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8960 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8961 }
8962 return Register();
8963}
8964
8965Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8966 if (RetVT.SimpleTy != MVT::nxv16i1)
8967 return Register();
8968 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8969 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8970 }
8971 return Register();
8972}
8973
8974Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8975 if (RetVT.SimpleTy != MVT::nxv16i8)
8976 return Register();
8977 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8978 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8979 }
8980 return Register();
8981}
8982
8983Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8984 if (RetVT.SimpleTy != MVT::nxv8i16)
8985 return Register();
8986 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8987 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8988 }
8989 return Register();
8990}
8991
8992Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8993 if (RetVT.SimpleTy != MVT::nxv4i32)
8994 return Register();
8995 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8996 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8997 }
8998 return Register();
8999}
9000
9001Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9002 if (RetVT.SimpleTy != MVT::nxv2i64)
9003 return Register();
9004 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9005 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9006 }
9007 return Register();
9008}
9009
9010Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9011 if (RetVT.SimpleTy != MVT::nxv2f16)
9012 return Register();
9013 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9014 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9015 }
9016 return Register();
9017}
9018
9019Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9020 if (RetVT.SimpleTy != MVT::nxv4f16)
9021 return Register();
9022 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9023 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9024 }
9025 return Register();
9026}
9027
9028Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9029 if (RetVT.SimpleTy != MVT::nxv8f16)
9030 return Register();
9031 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9032 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9033 }
9034 return Register();
9035}
9036
9037Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9038 if (RetVT.SimpleTy != MVT::nxv2bf16)
9039 return Register();
9040 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9041 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9042 }
9043 return Register();
9044}
9045
9046Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9047 if (RetVT.SimpleTy != MVT::nxv4bf16)
9048 return Register();
9049 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9050 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9051 }
9052 return Register();
9053}
9054
9055Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9056 if (RetVT.SimpleTy != MVT::nxv8bf16)
9057 return Register();
9058 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9059 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9060 }
9061 return Register();
9062}
9063
9064Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9065 if (RetVT.SimpleTy != MVT::nxv2f32)
9066 return Register();
9067 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9068 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9069 }
9070 return Register();
9071}
9072
9073Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9074 if (RetVT.SimpleTy != MVT::nxv4f32)
9075 return Register();
9076 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9077 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9078 }
9079 return Register();
9080}
9081
9082Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9083 if (RetVT.SimpleTy != MVT::nxv2f64)
9084 return Register();
9085 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9086 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9087 }
9088 return Register();
9089}
9090
9091Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9092 switch (VT.SimpleTy) {
9093 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9094 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9095 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9096 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9097 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9098 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9099 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9100 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9101 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9102 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9103 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9104 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9105 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9106 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9107 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9108 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9109 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9110 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9111 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9112 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9113 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9114 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9115 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9116 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9117 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9118 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9119 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9120 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9121 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9122 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9123 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9124 default: return Register();
9125 }
9126}
9127
9128// FastEmit functions for AArch64ISD::UZP2.
9129
9130Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9131 if (RetVT.SimpleTy != MVT::v8i8)
9132 return Register();
9133 if ((Subtarget->isNeonAvailable())) {
9134 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9135 }
9136 return Register();
9137}
9138
9139Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9140 if (RetVT.SimpleTy != MVT::v16i8)
9141 return Register();
9142 if ((Subtarget->isNeonAvailable())) {
9143 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9144 }
9145 return Register();
9146}
9147
9148Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9149 if (RetVT.SimpleTy != MVT::v4i16)
9150 return Register();
9151 if ((Subtarget->isNeonAvailable())) {
9152 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9153 }
9154 return Register();
9155}
9156
9157Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9158 if (RetVT.SimpleTy != MVT::v8i16)
9159 return Register();
9160 if ((Subtarget->isNeonAvailable())) {
9161 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9162 }
9163 return Register();
9164}
9165
9166Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9167 if (RetVT.SimpleTy != MVT::v2i32)
9168 return Register();
9169 if ((Subtarget->isNeonAvailable())) {
9170 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9171 }
9172 return Register();
9173}
9174
9175Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9176 if (RetVT.SimpleTy != MVT::v4i32)
9177 return Register();
9178 if ((Subtarget->isNeonAvailable())) {
9179 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9180 }
9181 return Register();
9182}
9183
9184Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9185 if (RetVT.SimpleTy != MVT::v2i64)
9186 return Register();
9187 if ((Subtarget->isNeonAvailable())) {
9188 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9189 }
9190 return Register();
9191}
9192
9193Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9194 if (RetVT.SimpleTy != MVT::v4f16)
9195 return Register();
9196 if ((Subtarget->isNeonAvailable())) {
9197 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9198 }
9199 return Register();
9200}
9201
9202Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9203 if (RetVT.SimpleTy != MVT::v8f16)
9204 return Register();
9205 if ((Subtarget->isNeonAvailable())) {
9206 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9207 }
9208 return Register();
9209}
9210
9211Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9212 if (RetVT.SimpleTy != MVT::v4bf16)
9213 return Register();
9214 if ((Subtarget->isNeonAvailable())) {
9215 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9216 }
9217 return Register();
9218}
9219
9220Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9221 if (RetVT.SimpleTy != MVT::v8bf16)
9222 return Register();
9223 if ((Subtarget->isNeonAvailable())) {
9224 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9225 }
9226 return Register();
9227}
9228
9229Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9230 if (RetVT.SimpleTy != MVT::v2f32)
9231 return Register();
9232 if ((Subtarget->isNeonAvailable())) {
9233 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9234 }
9235 return Register();
9236}
9237
9238Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9239 if (RetVT.SimpleTy != MVT::v4f32)
9240 return Register();
9241 if ((Subtarget->isNeonAvailable())) {
9242 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9243 }
9244 return Register();
9245}
9246
9247Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9248 if (RetVT.SimpleTy != MVT::v2f64)
9249 return Register();
9250 if ((Subtarget->isNeonAvailable())) {
9251 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9252 }
9253 return Register();
9254}
9255
9256Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9257 if (RetVT.SimpleTy != MVT::nxv2i1)
9258 return Register();
9259 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9260 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9261 }
9262 return Register();
9263}
9264
9265Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9266 if (RetVT.SimpleTy != MVT::nxv4i1)
9267 return Register();
9268 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9269 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9270 }
9271 return Register();
9272}
9273
9274Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9275 if (RetVT.SimpleTy != MVT::nxv8i1)
9276 return Register();
9277 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9278 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9279 }
9280 return Register();
9281}
9282
9283Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9284 if (RetVT.SimpleTy != MVT::nxv16i1)
9285 return Register();
9286 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9287 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9288 }
9289 return Register();
9290}
9291
9292Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9293 if (RetVT.SimpleTy != MVT::nxv16i8)
9294 return Register();
9295 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9296 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9297 }
9298 return Register();
9299}
9300
9301Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9302 if (RetVT.SimpleTy != MVT::nxv8i16)
9303 return Register();
9304 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9305 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9306 }
9307 return Register();
9308}
9309
9310Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9311 if (RetVT.SimpleTy != MVT::nxv4i32)
9312 return Register();
9313 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9314 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9315 }
9316 return Register();
9317}
9318
9319Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9320 if (RetVT.SimpleTy != MVT::nxv2i64)
9321 return Register();
9322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9323 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9324 }
9325 return Register();
9326}
9327
9328Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9329 if (RetVT.SimpleTy != MVT::nxv2f16)
9330 return Register();
9331 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9332 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9333 }
9334 return Register();
9335}
9336
9337Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9338 if (RetVT.SimpleTy != MVT::nxv4f16)
9339 return Register();
9340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9341 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9342 }
9343 return Register();
9344}
9345
9346Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9347 if (RetVT.SimpleTy != MVT::nxv8f16)
9348 return Register();
9349 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9350 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9351 }
9352 return Register();
9353}
9354
9355Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9356 if (RetVT.SimpleTy != MVT::nxv2bf16)
9357 return Register();
9358 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9359 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9360 }
9361 return Register();
9362}
9363
9364Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9365 if (RetVT.SimpleTy != MVT::nxv4bf16)
9366 return Register();
9367 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9368 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9369 }
9370 return Register();
9371}
9372
9373Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9374 if (RetVT.SimpleTy != MVT::nxv8bf16)
9375 return Register();
9376 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9377 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9378 }
9379 return Register();
9380}
9381
9382Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9383 if (RetVT.SimpleTy != MVT::nxv2f32)
9384 return Register();
9385 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9386 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9387 }
9388 return Register();
9389}
9390
9391Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9392 if (RetVT.SimpleTy != MVT::nxv4f32)
9393 return Register();
9394 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9395 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9396 }
9397 return Register();
9398}
9399
9400Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9401 if (RetVT.SimpleTy != MVT::nxv2f64)
9402 return Register();
9403 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9404 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9405 }
9406 return Register();
9407}
9408
9409Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9410 switch (VT.SimpleTy) {
9411 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9412 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9413 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9414 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9415 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9416 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9417 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9418 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9419 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9420 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9421 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9422 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9423 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9424 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9425 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9426 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9427 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9428 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9429 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9430 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9431 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9432 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9433 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9434 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9435 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9436 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9437 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9438 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9439 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9440 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9441 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9442 default: return Register();
9443 }
9444}
9445
9446// FastEmit functions for AArch64ISD::ZIP1.
9447
9448Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9449 if (RetVT.SimpleTy != MVT::v8i8)
9450 return Register();
9451 if ((Subtarget->isNeonAvailable())) {
9452 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9453 }
9454 return Register();
9455}
9456
9457Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9458 if (RetVT.SimpleTy != MVT::v16i8)
9459 return Register();
9460 if ((Subtarget->isNeonAvailable())) {
9461 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9462 }
9463 return Register();
9464}
9465
9466Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9467 if (RetVT.SimpleTy != MVT::v4i16)
9468 return Register();
9469 if ((Subtarget->isNeonAvailable())) {
9470 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9471 }
9472 return Register();
9473}
9474
9475Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9476 if (RetVT.SimpleTy != MVT::v8i16)
9477 return Register();
9478 if ((Subtarget->isNeonAvailable())) {
9479 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9480 }
9481 return Register();
9482}
9483
9484Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9485 if (RetVT.SimpleTy != MVT::v2i32)
9486 return Register();
9487 if ((Subtarget->isNeonAvailable())) {
9488 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9489 }
9490 return Register();
9491}
9492
9493Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9494 if (RetVT.SimpleTy != MVT::v4i32)
9495 return Register();
9496 if ((Subtarget->isNeonAvailable())) {
9497 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9498 }
9499 return Register();
9500}
9501
9502Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9503 if (RetVT.SimpleTy != MVT::v2i64)
9504 return Register();
9505 if ((Subtarget->isNeonAvailable())) {
9506 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9507 }
9508 return Register();
9509}
9510
9511Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9512 if (RetVT.SimpleTy != MVT::v4f16)
9513 return Register();
9514 if ((Subtarget->isNeonAvailable())) {
9515 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9516 }
9517 return Register();
9518}
9519
9520Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9521 if (RetVT.SimpleTy != MVT::v8f16)
9522 return Register();
9523 if ((Subtarget->isNeonAvailable())) {
9524 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9525 }
9526 return Register();
9527}
9528
9529Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9530 if (RetVT.SimpleTy != MVT::v4bf16)
9531 return Register();
9532 if ((Subtarget->isNeonAvailable())) {
9533 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9534 }
9535 return Register();
9536}
9537
9538Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9539 if (RetVT.SimpleTy != MVT::v8bf16)
9540 return Register();
9541 if ((Subtarget->isNeonAvailable())) {
9542 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9543 }
9544 return Register();
9545}
9546
9547Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9548 if (RetVT.SimpleTy != MVT::v2f32)
9549 return Register();
9550 if ((Subtarget->isNeonAvailable())) {
9551 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9552 }
9553 return Register();
9554}
9555
9556Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9557 if (RetVT.SimpleTy != MVT::v4f32)
9558 return Register();
9559 if ((Subtarget->isNeonAvailable())) {
9560 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9561 }
9562 return Register();
9563}
9564
9565Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9566 if (RetVT.SimpleTy != MVT::v2f64)
9567 return Register();
9568 if ((Subtarget->isNeonAvailable())) {
9569 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9570 }
9571 return Register();
9572}
9573
9574Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9575 if (RetVT.SimpleTy != MVT::nxv2i1)
9576 return Register();
9577 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9578 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9579 }
9580 return Register();
9581}
9582
9583Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9584 if (RetVT.SimpleTy != MVT::nxv4i1)
9585 return Register();
9586 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9587 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9588 }
9589 return Register();
9590}
9591
9592Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9593 if (RetVT.SimpleTy != MVT::nxv8i1)
9594 return Register();
9595 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9596 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9597 }
9598 return Register();
9599}
9600
9601Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9602 if (RetVT.SimpleTy != MVT::nxv16i1)
9603 return Register();
9604 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9605 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9606 }
9607 return Register();
9608}
9609
9610Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9611 if (RetVT.SimpleTy != MVT::nxv16i8)
9612 return Register();
9613 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9614 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9615 }
9616 return Register();
9617}
9618
9619Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9620 if (RetVT.SimpleTy != MVT::nxv8i16)
9621 return Register();
9622 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9623 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9624 }
9625 return Register();
9626}
9627
9628Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9629 if (RetVT.SimpleTy != MVT::nxv4i32)
9630 return Register();
9631 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9632 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9633 }
9634 return Register();
9635}
9636
9637Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9638 if (RetVT.SimpleTy != MVT::nxv2i64)
9639 return Register();
9640 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9641 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9642 }
9643 return Register();
9644}
9645
9646Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9647 if (RetVT.SimpleTy != MVT::nxv2f16)
9648 return Register();
9649 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9650 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9651 }
9652 return Register();
9653}
9654
9655Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9656 if (RetVT.SimpleTy != MVT::nxv4f16)
9657 return Register();
9658 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9659 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9660 }
9661 return Register();
9662}
9663
9664Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9665 if (RetVT.SimpleTy != MVT::nxv8f16)
9666 return Register();
9667 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9668 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9669 }
9670 return Register();
9671}
9672
9673Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9674 if (RetVT.SimpleTy != MVT::nxv2bf16)
9675 return Register();
9676 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9677 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9678 }
9679 return Register();
9680}
9681
9682Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9683 if (RetVT.SimpleTy != MVT::nxv4bf16)
9684 return Register();
9685 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9686 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9687 }
9688 return Register();
9689}
9690
9691Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9692 if (RetVT.SimpleTy != MVT::nxv8bf16)
9693 return Register();
9694 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9695 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9696 }
9697 return Register();
9698}
9699
9700Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9701 if (RetVT.SimpleTy != MVT::nxv2f32)
9702 return Register();
9703 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9704 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9705 }
9706 return Register();
9707}
9708
9709Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9710 if (RetVT.SimpleTy != MVT::nxv4f32)
9711 return Register();
9712 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9713 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9714 }
9715 return Register();
9716}
9717
9718Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9719 if (RetVT.SimpleTy != MVT::nxv2f64)
9720 return Register();
9721 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9722 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9723 }
9724 return Register();
9725}
9726
9727Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9728 switch (VT.SimpleTy) {
9729 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9730 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9731 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9732 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9733 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9734 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9735 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9736 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9737 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9738 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9739 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9740 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9741 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9742 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9743 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9744 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9745 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9746 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9747 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9748 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9749 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9750 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9751 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9752 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9753 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9754 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9755 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9756 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9757 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9758 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9759 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9760 default: return Register();
9761 }
9762}
9763
9764// FastEmit functions for AArch64ISD::ZIP2.
9765
9766Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9767 if (RetVT.SimpleTy != MVT::v8i8)
9768 return Register();
9769 if ((Subtarget->isNeonAvailable())) {
9770 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9771 }
9772 return Register();
9773}
9774
9775Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9776 if (RetVT.SimpleTy != MVT::v16i8)
9777 return Register();
9778 if ((Subtarget->isNeonAvailable())) {
9779 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9780 }
9781 return Register();
9782}
9783
9784Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9785 if (RetVT.SimpleTy != MVT::v4i16)
9786 return Register();
9787 if ((Subtarget->isNeonAvailable())) {
9788 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9789 }
9790 return Register();
9791}
9792
9793Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9794 if (RetVT.SimpleTy != MVT::v8i16)
9795 return Register();
9796 if ((Subtarget->isNeonAvailable())) {
9797 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9798 }
9799 return Register();
9800}
9801
9802Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9803 if (RetVT.SimpleTy != MVT::v2i32)
9804 return Register();
9805 if ((Subtarget->isNeonAvailable())) {
9806 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9807 }
9808 return Register();
9809}
9810
9811Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9812 if (RetVT.SimpleTy != MVT::v4i32)
9813 return Register();
9814 if ((Subtarget->isNeonAvailable())) {
9815 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9816 }
9817 return Register();
9818}
9819
9820Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9821 if (RetVT.SimpleTy != MVT::v2i64)
9822 return Register();
9823 if ((Subtarget->isNeonAvailable())) {
9824 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9825 }
9826 return Register();
9827}
9828
9829Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9830 if (RetVT.SimpleTy != MVT::v4f16)
9831 return Register();
9832 if ((Subtarget->isNeonAvailable())) {
9833 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9834 }
9835 return Register();
9836}
9837
9838Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9839 if (RetVT.SimpleTy != MVT::v8f16)
9840 return Register();
9841 if ((Subtarget->isNeonAvailable())) {
9842 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9843 }
9844 return Register();
9845}
9846
9847Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9848 if (RetVT.SimpleTy != MVT::v4bf16)
9849 return Register();
9850 if ((Subtarget->isNeonAvailable())) {
9851 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9852 }
9853 return Register();
9854}
9855
9856Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9857 if (RetVT.SimpleTy != MVT::v8bf16)
9858 return Register();
9859 if ((Subtarget->isNeonAvailable())) {
9860 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9861 }
9862 return Register();
9863}
9864
9865Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9866 if (RetVT.SimpleTy != MVT::v2f32)
9867 return Register();
9868 if ((Subtarget->isNeonAvailable())) {
9869 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9875 if (RetVT.SimpleTy != MVT::v4f32)
9876 return Register();
9877 if ((Subtarget->isNeonAvailable())) {
9878 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9879 }
9880 return Register();
9881}
9882
9883Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9884 if (RetVT.SimpleTy != MVT::v2f64)
9885 return Register();
9886 if ((Subtarget->isNeonAvailable())) {
9887 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9888 }
9889 return Register();
9890}
9891
9892Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9893 if (RetVT.SimpleTy != MVT::nxv2i1)
9894 return Register();
9895 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9896 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9897 }
9898 return Register();
9899}
9900
9901Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9902 if (RetVT.SimpleTy != MVT::nxv4i1)
9903 return Register();
9904 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9905 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9906 }
9907 return Register();
9908}
9909
9910Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9911 if (RetVT.SimpleTy != MVT::nxv8i1)
9912 return Register();
9913 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9914 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9915 }
9916 return Register();
9917}
9918
9919Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9920 if (RetVT.SimpleTy != MVT::nxv16i1)
9921 return Register();
9922 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9923 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9924 }
9925 return Register();
9926}
9927
9928Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9929 if (RetVT.SimpleTy != MVT::nxv16i8)
9930 return Register();
9931 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9932 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9933 }
9934 return Register();
9935}
9936
9937Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9938 if (RetVT.SimpleTy != MVT::nxv8i16)
9939 return Register();
9940 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9941 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9942 }
9943 return Register();
9944}
9945
9946Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9947 if (RetVT.SimpleTy != MVT::nxv4i32)
9948 return Register();
9949 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9950 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9951 }
9952 return Register();
9953}
9954
9955Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9956 if (RetVT.SimpleTy != MVT::nxv2i64)
9957 return Register();
9958 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9959 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9960 }
9961 return Register();
9962}
9963
9964Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9965 if (RetVT.SimpleTy != MVT::nxv2f16)
9966 return Register();
9967 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9968 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9969 }
9970 return Register();
9971}
9972
9973Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9974 if (RetVT.SimpleTy != MVT::nxv4f16)
9975 return Register();
9976 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9977 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9978 }
9979 return Register();
9980}
9981
9982Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9983 if (RetVT.SimpleTy != MVT::nxv8f16)
9984 return Register();
9985 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9986 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9987 }
9988 return Register();
9989}
9990
9991Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9992 if (RetVT.SimpleTy != MVT::nxv2bf16)
9993 return Register();
9994 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9995 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9996 }
9997 return Register();
9998}
9999
10000Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10001 if (RetVT.SimpleTy != MVT::nxv4bf16)
10002 return Register();
10003 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10004 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10005 }
10006 return Register();
10007}
10008
10009Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10010 if (RetVT.SimpleTy != MVT::nxv8bf16)
10011 return Register();
10012 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10013 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10014 }
10015 return Register();
10016}
10017
10018Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10019 if (RetVT.SimpleTy != MVT::nxv2f32)
10020 return Register();
10021 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10022 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10023 }
10024 return Register();
10025}
10026
10027Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10028 if (RetVT.SimpleTy != MVT::nxv4f32)
10029 return Register();
10030 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10031 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10032 }
10033 return Register();
10034}
10035
10036Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10037 if (RetVT.SimpleTy != MVT::nxv2f64)
10038 return Register();
10039 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10040 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10041 }
10042 return Register();
10043}
10044
10045Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10046 switch (VT.SimpleTy) {
10047 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
10048 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
10049 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
10050 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
10051 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
10052 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
10053 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
10054 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
10055 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
10056 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
10057 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
10058 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
10059 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
10060 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
10061 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10062 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10063 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10064 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
10065 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10066 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10067 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10068 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10069 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10070 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10071 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10072 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10073 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10074 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10075 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10076 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10077 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10078 default: return Register();
10079 }
10080}
10081
10082// FastEmit functions for ISD::ABDS.
10083
10084Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10085 if (RetVT.SimpleTy != MVT::v8i8)
10086 return Register();
10087 if ((Subtarget->isNeonAvailable())) {
10088 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10089 }
10090 return Register();
10091}
10092
10093Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10094 if (RetVT.SimpleTy != MVT::v16i8)
10095 return Register();
10096 if ((Subtarget->isNeonAvailable())) {
10097 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10098 }
10099 return Register();
10100}
10101
10102Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10103 if (RetVT.SimpleTy != MVT::v4i16)
10104 return Register();
10105 if ((Subtarget->isNeonAvailable())) {
10106 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10107 }
10108 return Register();
10109}
10110
10111Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10112 if (RetVT.SimpleTy != MVT::v8i16)
10113 return Register();
10114 if ((Subtarget->isNeonAvailable())) {
10115 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10116 }
10117 return Register();
10118}
10119
10120Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10121 if (RetVT.SimpleTy != MVT::v2i32)
10122 return Register();
10123 if ((Subtarget->isNeonAvailable())) {
10124 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10125 }
10126 return Register();
10127}
10128
10129Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10130 if (RetVT.SimpleTy != MVT::v4i32)
10131 return Register();
10132 if ((Subtarget->isNeonAvailable())) {
10133 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10134 }
10135 return Register();
10136}
10137
10138Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10139 switch (VT.SimpleTy) {
10140 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10141 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10142 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10143 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10144 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10145 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10146 default: return Register();
10147 }
10148}
10149
10150// FastEmit functions for ISD::ABDU.
10151
10152Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10153 if (RetVT.SimpleTy != MVT::v8i8)
10154 return Register();
10155 if ((Subtarget->isNeonAvailable())) {
10156 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10157 }
10158 return Register();
10159}
10160
10161Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10162 if (RetVT.SimpleTy != MVT::v16i8)
10163 return Register();
10164 if ((Subtarget->isNeonAvailable())) {
10165 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10166 }
10167 return Register();
10168}
10169
10170Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10171 if (RetVT.SimpleTy != MVT::v4i16)
10172 return Register();
10173 if ((Subtarget->isNeonAvailable())) {
10174 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10175 }
10176 return Register();
10177}
10178
10179Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10180 if (RetVT.SimpleTy != MVT::v8i16)
10181 return Register();
10182 if ((Subtarget->isNeonAvailable())) {
10183 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10184 }
10185 return Register();
10186}
10187
10188Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10189 if (RetVT.SimpleTy != MVT::v2i32)
10190 return Register();
10191 if ((Subtarget->isNeonAvailable())) {
10192 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10193 }
10194 return Register();
10195}
10196
10197Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10198 if (RetVT.SimpleTy != MVT::v4i32)
10199 return Register();
10200 if ((Subtarget->isNeonAvailable())) {
10201 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10202 }
10203 return Register();
10204}
10205
10206Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10207 switch (VT.SimpleTy) {
10208 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10209 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10210 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10211 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10212 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10213 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10214 default: return Register();
10215 }
10216}
10217
10218// FastEmit functions for ISD::ADD.
10219
10220Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10221 if (RetVT.SimpleTy != MVT::i32)
10222 return Register();
10223 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10224}
10225
10226Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10227 if (RetVT.SimpleTy != MVT::i64)
10228 return Register();
10229 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10230}
10231
10232Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10233 if (RetVT.SimpleTy != MVT::v8i8)
10234 return Register();
10235 if ((Subtarget->isNeonAvailable())) {
10236 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10237 }
10238 return Register();
10239}
10240
10241Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10242 if (RetVT.SimpleTy != MVT::v16i8)
10243 return Register();
10244 if ((Subtarget->isNeonAvailable())) {
10245 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10246 }
10247 return Register();
10248}
10249
10250Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10251 if (RetVT.SimpleTy != MVT::v4i16)
10252 return Register();
10253 if ((Subtarget->isNeonAvailable())) {
10254 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10255 }
10256 return Register();
10257}
10258
10259Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10260 if (RetVT.SimpleTy != MVT::v8i16)
10261 return Register();
10262 if ((Subtarget->isNeonAvailable())) {
10263 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10264 }
10265 return Register();
10266}
10267
10268Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10269 if (RetVT.SimpleTy != MVT::v2i32)
10270 return Register();
10271 if ((Subtarget->isNeonAvailable())) {
10272 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10273 }
10274 return Register();
10275}
10276
10277Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10278 if (RetVT.SimpleTy != MVT::v4i32)
10279 return Register();
10280 if ((Subtarget->isNeonAvailable())) {
10281 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10282 }
10283 return Register();
10284}
10285
10286Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10287 if (RetVT.SimpleTy != MVT::v1i64)
10288 return Register();
10289 if ((Subtarget->isNeonAvailable())) {
10290 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10291 }
10292 return Register();
10293}
10294
10295Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10296 if (RetVT.SimpleTy != MVT::v2i64)
10297 return Register();
10298 if ((Subtarget->isNeonAvailable())) {
10299 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10300 }
10301 return Register();
10302}
10303
10304Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10305 if (RetVT.SimpleTy != MVT::nxv16i8)
10306 return Register();
10307 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10308 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10309 }
10310 return Register();
10311}
10312
10313Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10314 if (RetVT.SimpleTy != MVT::nxv8i16)
10315 return Register();
10316 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10317 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10318 }
10319 return Register();
10320}
10321
10322Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10323 if (RetVT.SimpleTy != MVT::nxv4i32)
10324 return Register();
10325 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10326 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10327 }
10328 return Register();
10329}
10330
10331Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10332 if (RetVT.SimpleTy != MVT::nxv2i64)
10333 return Register();
10334 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10335 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10336 }
10337 return Register();
10338}
10339
10340Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10341 switch (VT.SimpleTy) {
10342 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10343 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10344 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10345 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10346 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10347 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10348 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10349 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10350 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10351 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10352 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10353 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10354 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10355 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10356 default: return Register();
10357 }
10358}
10359
10360// FastEmit functions for ISD::AND.
10361
10362Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
10363 if (RetVT.SimpleTy != MVT::i32)
10364 return Register();
10365 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10366}
10367
10368Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
10369 if (RetVT.SimpleTy != MVT::i64)
10370 return Register();
10371 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10372}
10373
10374Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10375 if (RetVT.SimpleTy != MVT::v8i8)
10376 return Register();
10377 if ((Subtarget->isNeonAvailable())) {
10378 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10379 }
10380 return Register();
10381}
10382
10383Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10384 if (RetVT.SimpleTy != MVT::v16i8)
10385 return Register();
10386 if ((Subtarget->isNeonAvailable())) {
10387 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10388 }
10389 return Register();
10390}
10391
10392Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10393 if (RetVT.SimpleTy != MVT::v4i16)
10394 return Register();
10395 if ((Subtarget->isNeonAvailable())) {
10396 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10397 }
10398 return Register();
10399}
10400
10401Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10402 if (RetVT.SimpleTy != MVT::v8i16)
10403 return Register();
10404 if ((Subtarget->isNeonAvailable())) {
10405 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10406 }
10407 return Register();
10408}
10409
10410Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10411 if (RetVT.SimpleTy != MVT::v2i32)
10412 return Register();
10413 if ((Subtarget->isNeonAvailable())) {
10414 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10415 }
10416 return Register();
10417}
10418
10419Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10420 if (RetVT.SimpleTy != MVT::v4i32)
10421 return Register();
10422 if ((Subtarget->isNeonAvailable())) {
10423 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10424 }
10425 return Register();
10426}
10427
10428Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
10429 if (RetVT.SimpleTy != MVT::v1i64)
10430 return Register();
10431 if ((Subtarget->isNeonAvailable())) {
10432 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10433 }
10434 return Register();
10435}
10436
10437Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10438 if (RetVT.SimpleTy != MVT::v2i64)
10439 return Register();
10440 if ((Subtarget->isNeonAvailable())) {
10441 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10442 }
10443 return Register();
10444}
10445
10446Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10447 if (RetVT.SimpleTy != MVT::nxv16i8)
10448 return Register();
10449 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10450 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10451 }
10452 return Register();
10453}
10454
10455Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10456 if (RetVT.SimpleTy != MVT::nxv8i16)
10457 return Register();
10458 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10459 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10460 }
10461 return Register();
10462}
10463
10464Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10465 if (RetVT.SimpleTy != MVT::nxv4i32)
10466 return Register();
10467 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10468 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10469 }
10470 return Register();
10471}
10472
10473Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10474 if (RetVT.SimpleTy != MVT::nxv2i64)
10475 return Register();
10476 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10477 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10478 }
10479 return Register();
10480}
10481
10482Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10483 switch (VT.SimpleTy) {
10484 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10485 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10486 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10487 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10488 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10489 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10490 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10491 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10492 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10493 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10494 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10495 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10496 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10497 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10498 default: return Register();
10499 }
10500}
10501
10502// FastEmit functions for ISD::AVGCEILS.
10503
10504Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10505 if (RetVT.SimpleTy != MVT::v8i8)
10506 return Register();
10507 if ((Subtarget->isNeonAvailable())) {
10508 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10509 }
10510 return Register();
10511}
10512
10513Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10514 if (RetVT.SimpleTy != MVT::v16i8)
10515 return Register();
10516 if ((Subtarget->isNeonAvailable())) {
10517 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10518 }
10519 return Register();
10520}
10521
10522Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10523 if (RetVT.SimpleTy != MVT::v4i16)
10524 return Register();
10525 if ((Subtarget->isNeonAvailable())) {
10526 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10527 }
10528 return Register();
10529}
10530
10531Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10532 if (RetVT.SimpleTy != MVT::v8i16)
10533 return Register();
10534 if ((Subtarget->isNeonAvailable())) {
10535 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10536 }
10537 return Register();
10538}
10539
10540Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10541 if (RetVT.SimpleTy != MVT::v2i32)
10542 return Register();
10543 if ((Subtarget->isNeonAvailable())) {
10544 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10545 }
10546 return Register();
10547}
10548
10549Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10550 if (RetVT.SimpleTy != MVT::v4i32)
10551 return Register();
10552 if ((Subtarget->isNeonAvailable())) {
10553 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10554 }
10555 return Register();
10556}
10557
10558Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10559 switch (VT.SimpleTy) {
10560 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10561 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10562 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10563 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10564 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10565 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10566 default: return Register();
10567 }
10568}
10569
10570// FastEmit functions for ISD::AVGCEILU.
10571
10572Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10573 if (RetVT.SimpleTy != MVT::v8i8)
10574 return Register();
10575 if ((Subtarget->isNeonAvailable())) {
10576 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10577 }
10578 return Register();
10579}
10580
10581Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10582 if (RetVT.SimpleTy != MVT::v16i8)
10583 return Register();
10584 if ((Subtarget->isNeonAvailable())) {
10585 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10586 }
10587 return Register();
10588}
10589
10590Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10591 if (RetVT.SimpleTy != MVT::v4i16)
10592 return Register();
10593 if ((Subtarget->isNeonAvailable())) {
10594 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10595 }
10596 return Register();
10597}
10598
10599Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10600 if (RetVT.SimpleTy != MVT::v8i16)
10601 return Register();
10602 if ((Subtarget->isNeonAvailable())) {
10603 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10604 }
10605 return Register();
10606}
10607
10608Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10609 if (RetVT.SimpleTy != MVT::v2i32)
10610 return Register();
10611 if ((Subtarget->isNeonAvailable())) {
10612 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10613 }
10614 return Register();
10615}
10616
10617Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10618 if (RetVT.SimpleTy != MVT::v4i32)
10619 return Register();
10620 if ((Subtarget->isNeonAvailable())) {
10621 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10622 }
10623 return Register();
10624}
10625
10626Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10627 switch (VT.SimpleTy) {
10628 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10629 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10630 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10631 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10632 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10633 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10634 default: return Register();
10635 }
10636}
10637
10638// FastEmit functions for ISD::AVGFLOORS.
10639
10640Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10641 if (RetVT.SimpleTy != MVT::v8i8)
10642 return Register();
10643 if ((Subtarget->isNeonAvailable())) {
10644 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10645 }
10646 return Register();
10647}
10648
10649Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10650 if (RetVT.SimpleTy != MVT::v16i8)
10651 return Register();
10652 if ((Subtarget->isNeonAvailable())) {
10653 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10654 }
10655 return Register();
10656}
10657
10658Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10659 if (RetVT.SimpleTy != MVT::v4i16)
10660 return Register();
10661 if ((Subtarget->isNeonAvailable())) {
10662 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10663 }
10664 return Register();
10665}
10666
10667Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10668 if (RetVT.SimpleTy != MVT::v8i16)
10669 return Register();
10670 if ((Subtarget->isNeonAvailable())) {
10671 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10672 }
10673 return Register();
10674}
10675
10676Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10677 if (RetVT.SimpleTy != MVT::v2i32)
10678 return Register();
10679 if ((Subtarget->isNeonAvailable())) {
10680 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10681 }
10682 return Register();
10683}
10684
10685Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10686 if (RetVT.SimpleTy != MVT::v4i32)
10687 return Register();
10688 if ((Subtarget->isNeonAvailable())) {
10689 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10690 }
10691 return Register();
10692}
10693
10694Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10695 switch (VT.SimpleTy) {
10696 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10697 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10698 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10699 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10700 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10701 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10702 default: return Register();
10703 }
10704}
10705
10706// FastEmit functions for ISD::AVGFLOORU.
10707
10708Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10709 if (RetVT.SimpleTy != MVT::v8i8)
10710 return Register();
10711 if ((Subtarget->isNeonAvailable())) {
10712 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10713 }
10714 return Register();
10715}
10716
10717Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10718 if (RetVT.SimpleTy != MVT::v16i8)
10719 return Register();
10720 if ((Subtarget->isNeonAvailable())) {
10721 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10722 }
10723 return Register();
10724}
10725
10726Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10727 if (RetVT.SimpleTy != MVT::v4i16)
10728 return Register();
10729 if ((Subtarget->isNeonAvailable())) {
10730 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10731 }
10732 return Register();
10733}
10734
10735Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10736 if (RetVT.SimpleTy != MVT::v8i16)
10737 return Register();
10738 if ((Subtarget->isNeonAvailable())) {
10739 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10740 }
10741 return Register();
10742}
10743
10744Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10745 if (RetVT.SimpleTy != MVT::v2i32)
10746 return Register();
10747 if ((Subtarget->isNeonAvailable())) {
10748 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10749 }
10750 return Register();
10751}
10752
10753Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10754 if (RetVT.SimpleTy != MVT::v4i32)
10755 return Register();
10756 if ((Subtarget->isNeonAvailable())) {
10757 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10758 }
10759 return Register();
10760}
10761
10762Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10763 switch (VT.SimpleTy) {
10764 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10765 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10766 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10767 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10768 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10769 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10770 default: return Register();
10771 }
10772}
10773
10774// FastEmit functions for ISD::CLMUL.
10775
10776Register fastEmit_ISD_CLMUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10777 if (RetVT.SimpleTy != MVT::v8i8)
10778 return Register();
10779 if ((Subtarget->isNeonAvailable())) {
10780 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10781 }
10782 return Register();
10783}
10784
10785Register fastEmit_ISD_CLMUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10786 if (RetVT.SimpleTy != MVT::v16i8)
10787 return Register();
10788 if ((Subtarget->isNeonAvailable())) {
10789 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10790 }
10791 return Register();
10792}
10793
10794Register fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10795 if (RetVT.SimpleTy != MVT::nxv16i8)
10796 return Register();
10797 if ((Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2() || Subtarget->hasSME()))) {
10798 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMUL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10799 }
10800 return Register();
10801}
10802
10803Register fastEmit_ISD_CLMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10804 switch (VT.SimpleTy) {
10805 case MVT::v8i8: return fastEmit_ISD_CLMUL_MVT_v8i8_rr(RetVT, Op0, Op1);
10806 case MVT::v16i8: return fastEmit_ISD_CLMUL_MVT_v16i8_rr(RetVT, Op0, Op1);
10807 case MVT::nxv16i8: return fastEmit_ISD_CLMUL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10808 default: return Register();
10809 }
10810}
10811
10812// FastEmit functions for ISD::CONCAT_VECTORS.
10813
10814Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10815 if (RetVT.SimpleTy != MVT::nxv2i1)
10816 return Register();
10817 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10818 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10819 }
10820 return Register();
10821}
10822
10823Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10824 if (RetVT.SimpleTy != MVT::nxv4i1)
10825 return Register();
10826 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10827 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10828 }
10829 return Register();
10830}
10831
10832Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10833 if (RetVT.SimpleTy != MVT::nxv8i1)
10834 return Register();
10835 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10836 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10837 }
10838 return Register();
10839}
10840
10841Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10842 if (RetVT.SimpleTy != MVT::nxv16i1)
10843 return Register();
10844 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10845 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10846 }
10847 return Register();
10848}
10849
10850Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10851 if (RetVT.SimpleTy != MVT::nxv4f16)
10852 return Register();
10853 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10854 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10855 }
10856 return Register();
10857}
10858
10859Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10860 if (RetVT.SimpleTy != MVT::nxv8f16)
10861 return Register();
10862 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10863 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10864 }
10865 return Register();
10866}
10867
10868Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10869 if (RetVT.SimpleTy != MVT::nxv4bf16)
10870 return Register();
10871 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10872 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10873 }
10874 return Register();
10875}
10876
10877Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10878 if (RetVT.SimpleTy != MVT::nxv8bf16)
10879 return Register();
10880 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10881 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10882 }
10883 return Register();
10884}
10885
10886Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10887 if (RetVT.SimpleTy != MVT::nxv4f32)
10888 return Register();
10889 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10890 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10891 }
10892 return Register();
10893}
10894
10895Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10896 switch (VT.SimpleTy) {
10897 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10898 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10899 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10900 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10901 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10902 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10903 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10904 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10905 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10906 default: return Register();
10907 }
10908}
10909
10910// FastEmit functions for ISD::FADD.
10911
10912Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10913 if (RetVT.SimpleTy != MVT::f16)
10914 return Register();
10915 if ((Subtarget->hasFullFP16())) {
10916 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10917 }
10918 return Register();
10919}
10920
10921Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10922 if (RetVT.SimpleTy != MVT::f32)
10923 return Register();
10924 if ((Subtarget->hasFPARMv8())) {
10925 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10926 }
10927 return Register();
10928}
10929
10930Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10931 if (RetVT.SimpleTy != MVT::f64)
10932 return Register();
10933 if ((Subtarget->hasFPARMv8())) {
10934 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10935 }
10936 return Register();
10937}
10938
10939Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10940 if (RetVT.SimpleTy != MVT::v4f16)
10941 return Register();
10942 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10943 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10944 }
10945 return Register();
10946}
10947
10948Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10949 if (RetVT.SimpleTy != MVT::v8f16)
10950 return Register();
10951 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10952 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10953 }
10954 return Register();
10955}
10956
10957Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10958 if (RetVT.SimpleTy != MVT::v2f32)
10959 return Register();
10960 if ((Subtarget->isNeonAvailable())) {
10961 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10962 }
10963 return Register();
10964}
10965
10966Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10967 if (RetVT.SimpleTy != MVT::v4f32)
10968 return Register();
10969 if ((Subtarget->isNeonAvailable())) {
10970 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10971 }
10972 return Register();
10973}
10974
10975Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10976 if (RetVT.SimpleTy != MVT::v2f64)
10977 return Register();
10978 if ((Subtarget->isNeonAvailable())) {
10979 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10980 }
10981 return Register();
10982}
10983
10984Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10985 if (RetVT.SimpleTy != MVT::nxv8f16)
10986 return Register();
10987 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10988 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10989 }
10990 return Register();
10991}
10992
10993Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10994 if (RetVT.SimpleTy != MVT::nxv8bf16)
10995 return Register();
10996 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
10997 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10998 }
10999 return Register();
11000}
11001
11002Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11003 if (RetVT.SimpleTy != MVT::nxv4f32)
11004 return Register();
11005 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11006 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11007 }
11008 return Register();
11009}
11010
11011Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11012 if (RetVT.SimpleTy != MVT::nxv2f64)
11013 return Register();
11014 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11015 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11016 }
11017 return Register();
11018}
11019
11020Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11021 switch (VT.SimpleTy) {
11022 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11023 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11024 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11025 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
11026 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11027 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
11028 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11029 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11030 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11031 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11032 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11033 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11034 default: return Register();
11035 }
11036}
11037
11038// FastEmit functions for ISD::FDIV.
11039
11040Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11041 if (RetVT.SimpleTy != MVT::f16)
11042 return Register();
11043 if ((Subtarget->hasFullFP16())) {
11044 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11045 }
11046 return Register();
11047}
11048
11049Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11050 if (RetVT.SimpleTy != MVT::f32)
11051 return Register();
11052 if ((Subtarget->hasFPARMv8())) {
11053 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11054 }
11055 return Register();
11056}
11057
11058Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11059 if (RetVT.SimpleTy != MVT::f64)
11060 return Register();
11061 if ((Subtarget->hasFPARMv8())) {
11062 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11063 }
11064 return Register();
11065}
11066
11067Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11068 if (RetVT.SimpleTy != MVT::v4f16)
11069 return Register();
11070 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11071 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11072 }
11073 return Register();
11074}
11075
11076Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11077 if (RetVT.SimpleTy != MVT::v8f16)
11078 return Register();
11079 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11080 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11081 }
11082 return Register();
11083}
11084
11085Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11086 if (RetVT.SimpleTy != MVT::v2f32)
11087 return Register();
11088 if ((Subtarget->isNeonAvailable())) {
11089 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11090 }
11091 return Register();
11092}
11093
11094Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11095 if (RetVT.SimpleTy != MVT::v4f32)
11096 return Register();
11097 if ((Subtarget->isNeonAvailable())) {
11098 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11099 }
11100 return Register();
11101}
11102
11103Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11104 if (RetVT.SimpleTy != MVT::v2f64)
11105 return Register();
11106 if ((Subtarget->isNeonAvailable())) {
11107 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11108 }
11109 return Register();
11110}
11111
11112Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11113 switch (VT.SimpleTy) {
11114 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11115 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11116 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11117 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
11118 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11119 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
11120 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11121 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11122 default: return Register();
11123 }
11124}
11125
11126// FastEmit functions for ISD::FMAXIMUM.
11127
11128Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11129 if (RetVT.SimpleTy != MVT::f16)
11130 return Register();
11131 if ((Subtarget->hasFullFP16())) {
11132 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11133 }
11134 return Register();
11135}
11136
11137Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11138 if (RetVT.SimpleTy != MVT::f32)
11139 return Register();
11140 if ((Subtarget->hasFPARMv8())) {
11141 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11142 }
11143 return Register();
11144}
11145
11146Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11147 if (RetVT.SimpleTy != MVT::f64)
11148 return Register();
11149 if ((Subtarget->hasFPARMv8())) {
11150 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11151 }
11152 return Register();
11153}
11154
11155Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11156 if (RetVT.SimpleTy != MVT::v4f16)
11157 return Register();
11158 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11159 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11160 }
11161 return Register();
11162}
11163
11164Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11165 if (RetVT.SimpleTy != MVT::v8f16)
11166 return Register();
11167 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11168 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11169 }
11170 return Register();
11171}
11172
11173Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11174 if (RetVT.SimpleTy != MVT::v2f32)
11175 return Register();
11176 if ((Subtarget->isNeonAvailable())) {
11177 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11178 }
11179 return Register();
11180}
11181
11182Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11183 if (RetVT.SimpleTy != MVT::v4f32)
11184 return Register();
11185 if ((Subtarget->isNeonAvailable())) {
11186 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11187 }
11188 return Register();
11189}
11190
11191Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11192 if (RetVT.SimpleTy != MVT::v1f64)
11193 return Register();
11194 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11195}
11196
11197Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11198 if (RetVT.SimpleTy != MVT::v2f64)
11199 return Register();
11200 if ((Subtarget->isNeonAvailable())) {
11201 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11202 }
11203 return Register();
11204}
11205
11206Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11207 switch (VT.SimpleTy) {
11208 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11209 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11210 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11211 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11212 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11213 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11214 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11215 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11216 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11217 default: return Register();
11218 }
11219}
11220
11221// FastEmit functions for ISD::FMAXNUM.
11222
11223Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11224 if (RetVT.SimpleTy != MVT::f16)
11225 return Register();
11226 if ((Subtarget->hasFullFP16())) {
11227 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11228 }
11229 return Register();
11230}
11231
11232Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11233 if (RetVT.SimpleTy != MVT::f32)
11234 return Register();
11235 if ((Subtarget->hasFPARMv8())) {
11236 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11237 }
11238 return Register();
11239}
11240
11241Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11242 if (RetVT.SimpleTy != MVT::f64)
11243 return Register();
11244 if ((Subtarget->hasFPARMv8())) {
11245 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11246 }
11247 return Register();
11248}
11249
11250Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11251 if (RetVT.SimpleTy != MVT::v4f16)
11252 return Register();
11253 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11254 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11255 }
11256 return Register();
11257}
11258
11259Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11260 if (RetVT.SimpleTy != MVT::v8f16)
11261 return Register();
11262 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11263 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11264 }
11265 return Register();
11266}
11267
11268Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11269 if (RetVT.SimpleTy != MVT::v2f32)
11270 return Register();
11271 if ((Subtarget->isNeonAvailable())) {
11272 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11273 }
11274 return Register();
11275}
11276
11277Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11278 if (RetVT.SimpleTy != MVT::v4f32)
11279 return Register();
11280 if ((Subtarget->isNeonAvailable())) {
11281 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11282 }
11283 return Register();
11284}
11285
11286Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11287 if (RetVT.SimpleTy != MVT::v1f64)
11288 return Register();
11289 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11290}
11291
11292Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11293 if (RetVT.SimpleTy != MVT::v2f64)
11294 return Register();
11295 if ((Subtarget->isNeonAvailable())) {
11296 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11297 }
11298 return Register();
11299}
11300
11301Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11302 switch (VT.SimpleTy) {
11303 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11304 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11305 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11306 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11307 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11308 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11309 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11310 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11311 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11312 default: return Register();
11313 }
11314}
11315
11316// FastEmit functions for ISD::FMAXNUM_IEEE.
11317
11318Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11319 if (RetVT.SimpleTy != MVT::f16)
11320 return Register();
11321 if ((Subtarget->hasFullFP16())) {
11322 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11323 }
11324 return Register();
11325}
11326
11327Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11328 if (RetVT.SimpleTy != MVT::f32)
11329 return Register();
11330 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11331}
11332
11333Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11334 if (RetVT.SimpleTy != MVT::f64)
11335 return Register();
11336 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11337}
11338
11339Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11340 if (RetVT.SimpleTy != MVT::v4f16)
11341 return Register();
11342 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11343 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11344 }
11345 return Register();
11346}
11347
11348Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11349 if (RetVT.SimpleTy != MVT::v8f16)
11350 return Register();
11351 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11353 }
11354 return Register();
11355}
11356
11357Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11358 if (RetVT.SimpleTy != MVT::v2f32)
11359 return Register();
11360 if ((Subtarget->isNeonAvailable())) {
11361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11362 }
11363 return Register();
11364}
11365
11366Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11367 if (RetVT.SimpleTy != MVT::v4f32)
11368 return Register();
11369 if ((Subtarget->isNeonAvailable())) {
11370 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11371 }
11372 return Register();
11373}
11374
11375Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11376 if (RetVT.SimpleTy != MVT::v2f64)
11377 return Register();
11378 if ((Subtarget->isNeonAvailable())) {
11379 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11380 }
11381 return Register();
11382}
11383
11384Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11385 switch (VT.SimpleTy) {
11386 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11387 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11388 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11389 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11390 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11391 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11392 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11393 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11394 default: return Register();
11395 }
11396}
11397
11398// FastEmit functions for ISD::FMINIMUM.
11399
11400Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11401 if (RetVT.SimpleTy != MVT::f16)
11402 return Register();
11403 if ((Subtarget->hasFullFP16())) {
11404 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11405 }
11406 return Register();
11407}
11408
11409Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11410 if (RetVT.SimpleTy != MVT::f32)
11411 return Register();
11412 if ((Subtarget->hasFPARMv8())) {
11413 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11414 }
11415 return Register();
11416}
11417
11418Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11419 if (RetVT.SimpleTy != MVT::f64)
11420 return Register();
11421 if ((Subtarget->hasFPARMv8())) {
11422 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11423 }
11424 return Register();
11425}
11426
11427Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11428 if (RetVT.SimpleTy != MVT::v4f16)
11429 return Register();
11430 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11431 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11432 }
11433 return Register();
11434}
11435
11436Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11437 if (RetVT.SimpleTy != MVT::v8f16)
11438 return Register();
11439 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11440 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11441 }
11442 return Register();
11443}
11444
11445Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11446 if (RetVT.SimpleTy != MVT::v2f32)
11447 return Register();
11448 if ((Subtarget->isNeonAvailable())) {
11449 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11450 }
11451 return Register();
11452}
11453
11454Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11455 if (RetVT.SimpleTy != MVT::v4f32)
11456 return Register();
11457 if ((Subtarget->isNeonAvailable())) {
11458 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11459 }
11460 return Register();
11461}
11462
11463Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11464 if (RetVT.SimpleTy != MVT::v1f64)
11465 return Register();
11466 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11467}
11468
11469Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11470 if (RetVT.SimpleTy != MVT::v2f64)
11471 return Register();
11472 if ((Subtarget->isNeonAvailable())) {
11473 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11474 }
11475 return Register();
11476}
11477
11478Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11479 switch (VT.SimpleTy) {
11480 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11481 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11482 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11483 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11484 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11485 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11486 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11487 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11488 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11489 default: return Register();
11490 }
11491}
11492
11493// FastEmit functions for ISD::FMINNUM.
11494
11495Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11496 if (RetVT.SimpleTy != MVT::f16)
11497 return Register();
11498 if ((Subtarget->hasFullFP16())) {
11499 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11500 }
11501 return Register();
11502}
11503
11504Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11505 if (RetVT.SimpleTy != MVT::f32)
11506 return Register();
11507 if ((Subtarget->hasFPARMv8())) {
11508 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11509 }
11510 return Register();
11511}
11512
11513Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11514 if (RetVT.SimpleTy != MVT::f64)
11515 return Register();
11516 if ((Subtarget->hasFPARMv8())) {
11517 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11518 }
11519 return Register();
11520}
11521
11522Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11523 if (RetVT.SimpleTy != MVT::v4f16)
11524 return Register();
11525 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11526 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11527 }
11528 return Register();
11529}
11530
11531Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11532 if (RetVT.SimpleTy != MVT::v8f16)
11533 return Register();
11534 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11535 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11536 }
11537 return Register();
11538}
11539
11540Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11541 if (RetVT.SimpleTy != MVT::v2f32)
11542 return Register();
11543 if ((Subtarget->isNeonAvailable())) {
11544 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11545 }
11546 return Register();
11547}
11548
11549Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11550 if (RetVT.SimpleTy != MVT::v4f32)
11551 return Register();
11552 if ((Subtarget->isNeonAvailable())) {
11553 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11554 }
11555 return Register();
11556}
11557
11558Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11559 if (RetVT.SimpleTy != MVT::v1f64)
11560 return Register();
11561 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11562}
11563
11564Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11565 if (RetVT.SimpleTy != MVT::v2f64)
11566 return Register();
11567 if ((Subtarget->isNeonAvailable())) {
11568 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11569 }
11570 return Register();
11571}
11572
11573Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11574 switch (VT.SimpleTy) {
11575 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11576 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11577 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11578 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11579 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11580 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11581 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11582 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11583 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11584 default: return Register();
11585 }
11586}
11587
11588// FastEmit functions for ISD::FMINNUM_IEEE.
11589
11590Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11591 if (RetVT.SimpleTy != MVT::f16)
11592 return Register();
11593 if ((Subtarget->hasFullFP16())) {
11594 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11595 }
11596 return Register();
11597}
11598
11599Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11600 if (RetVT.SimpleTy != MVT::f32)
11601 return Register();
11602 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11603}
11604
11605Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11606 if (RetVT.SimpleTy != MVT::f64)
11607 return Register();
11608 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11609}
11610
11611Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11612 if (RetVT.SimpleTy != MVT::v4f16)
11613 return Register();
11614 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11615 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11616 }
11617 return Register();
11618}
11619
11620Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11621 if (RetVT.SimpleTy != MVT::v8f16)
11622 return Register();
11623 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11624 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11625 }
11626 return Register();
11627}
11628
11629Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11630 if (RetVT.SimpleTy != MVT::v2f32)
11631 return Register();
11632 if ((Subtarget->isNeonAvailable())) {
11633 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11634 }
11635 return Register();
11636}
11637
11638Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11639 if (RetVT.SimpleTy != MVT::v4f32)
11640 return Register();
11641 if ((Subtarget->isNeonAvailable())) {
11642 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11643 }
11644 return Register();
11645}
11646
11647Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11648 if (RetVT.SimpleTy != MVT::v2f64)
11649 return Register();
11650 if ((Subtarget->isNeonAvailable())) {
11651 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11652 }
11653 return Register();
11654}
11655
11656Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11657 switch (VT.SimpleTy) {
11658 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11659 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11660 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11661 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11662 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11663 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11664 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11665 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11666 default: return Register();
11667 }
11668}
11669
11670// FastEmit functions for ISD::FMUL.
11671
11672Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11673 if (RetVT.SimpleTy != MVT::f16)
11674 return Register();
11675 if ((Subtarget->hasFullFP16())) {
11676 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11677 }
11678 return Register();
11679}
11680
11681Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11682 if (RetVT.SimpleTy != MVT::f32)
11683 return Register();
11684 if ((Subtarget->hasFPARMv8())) {
11685 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11686 }
11687 return Register();
11688}
11689
11690Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11691 if (RetVT.SimpleTy != MVT::f64)
11692 return Register();
11693 if ((Subtarget->hasFPARMv8())) {
11694 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11695 }
11696 return Register();
11697}
11698
11699Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11700 if (RetVT.SimpleTy != MVT::v4f16)
11701 return Register();
11702 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11703 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11704 }
11705 return Register();
11706}
11707
11708Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11709 if (RetVT.SimpleTy != MVT::v8f16)
11710 return Register();
11711 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11712 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11713 }
11714 return Register();
11715}
11716
11717Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11718 if (RetVT.SimpleTy != MVT::v2f32)
11719 return Register();
11720 if ((Subtarget->isNeonAvailable())) {
11721 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11722 }
11723 return Register();
11724}
11725
11726Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11727 if (RetVT.SimpleTy != MVT::v4f32)
11728 return Register();
11729 if ((Subtarget->isNeonAvailable())) {
11730 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11731 }
11732 return Register();
11733}
11734
11735Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11736 if (RetVT.SimpleTy != MVT::v2f64)
11737 return Register();
11738 if ((Subtarget->isNeonAvailable())) {
11739 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11740 }
11741 return Register();
11742}
11743
11744Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11745 if (RetVT.SimpleTy != MVT::nxv8f16)
11746 return Register();
11747 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11748 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11749 }
11750 return Register();
11751}
11752
11753Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11754 if (RetVT.SimpleTy != MVT::nxv8bf16)
11755 return Register();
11756 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11757 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11758 }
11759 return Register();
11760}
11761
11762Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11763 if (RetVT.SimpleTy != MVT::nxv4f32)
11764 return Register();
11765 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11766 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11767 }
11768 return Register();
11769}
11770
11771Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11772 if (RetVT.SimpleTy != MVT::nxv2f64)
11773 return Register();
11774 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11775 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11776 }
11777 return Register();
11778}
11779
11780Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11781 switch (VT.SimpleTy) {
11782 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11783 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11784 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11785 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11786 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11787 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11788 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11789 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11790 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11791 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11792 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11793 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11794 default: return Register();
11795 }
11796}
11797
11798// FastEmit functions for ISD::FSUB.
11799
11800Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11801 if (RetVT.SimpleTy != MVT::f16)
11802 return Register();
11803 if ((Subtarget->hasFullFP16())) {
11804 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11805 }
11806 return Register();
11807}
11808
11809Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11810 if (RetVT.SimpleTy != MVT::f32)
11811 return Register();
11812 if ((Subtarget->hasFPARMv8())) {
11813 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11814 }
11815 return Register();
11816}
11817
11818Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11819 if (RetVT.SimpleTy != MVT::f64)
11820 return Register();
11821 if ((Subtarget->hasFPARMv8())) {
11822 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11823 }
11824 return Register();
11825}
11826
11827Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11828 if (RetVT.SimpleTy != MVT::v4f16)
11829 return Register();
11830 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11831 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11832 }
11833 return Register();
11834}
11835
11836Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11837 if (RetVT.SimpleTy != MVT::v8f16)
11838 return Register();
11839 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11840 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11841 }
11842 return Register();
11843}
11844
11845Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11846 if (RetVT.SimpleTy != MVT::v2f32)
11847 return Register();
11848 if ((Subtarget->isNeonAvailable())) {
11849 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11850 }
11851 return Register();
11852}
11853
11854Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11855 if (RetVT.SimpleTy != MVT::v4f32)
11856 return Register();
11857 if ((Subtarget->isNeonAvailable())) {
11858 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11859 }
11860 return Register();
11861}
11862
11863Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11864 if (RetVT.SimpleTy != MVT::v2f64)
11865 return Register();
11866 if ((Subtarget->isNeonAvailable())) {
11867 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11868 }
11869 return Register();
11870}
11871
11872Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11873 if (RetVT.SimpleTy != MVT::nxv8f16)
11874 return Register();
11875 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11876 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11877 }
11878 return Register();
11879}
11880
11881Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11882 if (RetVT.SimpleTy != MVT::nxv8bf16)
11883 return Register();
11884 if ((Subtarget->isNonStreamingSVEorSME2Available()) && (Subtarget->hasSVEB16B16())) {
11885 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11886 }
11887 return Register();
11888}
11889
11890Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11891 if (RetVT.SimpleTy != MVT::nxv4f32)
11892 return Register();
11893 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11894 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11895 }
11896 return Register();
11897}
11898
11899Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11900 if (RetVT.SimpleTy != MVT::nxv2f64)
11901 return Register();
11902 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11903 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11904 }
11905 return Register();
11906}
11907
11908Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11909 switch (VT.SimpleTy) {
11910 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11911 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11912 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11913 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11914 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11915 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11916 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11917 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11918 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11919 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11920 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11921 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11922 default: return Register();
11923 }
11924}
11925
11926// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11927
11928Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11929 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11930 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11931 }
11932 return Register();
11933}
11934
11935Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11937 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11938 }
11939 return Register();
11940}
11941
11942Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11943 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11944 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11945 }
11946 return Register();
11947}
11948
11949Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11950 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11951 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11952 }
11953 return Register();
11954}
11955
11956Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11957switch (RetVT.SimpleTy) {
11958 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11959 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11960 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11961 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11962 default: return Register();
11963}
11964}
11965
11966Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11967 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11968 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11969 }
11970 return Register();
11971}
11972
11973Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11974 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11975 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11976 }
11977 return Register();
11978}
11979
11980Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11981 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11982 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11983 }
11984 return Register();
11985}
11986
11987Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11988 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11989 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11990 }
11991 return Register();
11992}
11993
11994Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11995switch (RetVT.SimpleTy) {
11996 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11997 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11998 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11999 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
12000 default: return Register();
12001}
12002}
12003
12004Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12005 switch (VT.SimpleTy) {
12006 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
12007 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
12008 default: return Register();
12009 }
12010}
12011
12012// FastEmit functions for ISD::MUL.
12013
12014Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12015 if (RetVT.SimpleTy != MVT::v8i8)
12016 return Register();
12017 if ((Subtarget->isNeonAvailable())) {
12018 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12019 }
12020 return Register();
12021}
12022
12023Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12024 if (RetVT.SimpleTy != MVT::v16i8)
12025 return Register();
12026 if ((Subtarget->isNeonAvailable())) {
12027 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12028 }
12029 return Register();
12030}
12031
12032Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12033 if (RetVT.SimpleTy != MVT::v4i16)
12034 return Register();
12035 if ((Subtarget->isNeonAvailable())) {
12036 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12037 }
12038 return Register();
12039}
12040
12041Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12042 if (RetVT.SimpleTy != MVT::v8i16)
12043 return Register();
12044 if ((Subtarget->isNeonAvailable())) {
12045 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12046 }
12047 return Register();
12048}
12049
12050Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12051 if (RetVT.SimpleTy != MVT::v2i32)
12052 return Register();
12053 if ((Subtarget->isNeonAvailable())) {
12054 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12055 }
12056 return Register();
12057}
12058
12059Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12060 if (RetVT.SimpleTy != MVT::v4i32)
12061 return Register();
12062 if ((Subtarget->isNeonAvailable())) {
12063 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12064 }
12065 return Register();
12066}
12067
12068Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12069 switch (VT.SimpleTy) {
12070 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
12071 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
12072 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
12073 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
12074 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
12075 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
12076 default: return Register();
12077 }
12078}
12079
12080// FastEmit functions for ISD::MULHS.
12081
12082Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12083 if (RetVT.SimpleTy != MVT::i64)
12084 return Register();
12085 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12086}
12087
12088Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12089 switch (VT.SimpleTy) {
12090 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
12091 default: return Register();
12092 }
12093}
12094
12095// FastEmit functions for ISD::MULHU.
12096
12097Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12098 if (RetVT.SimpleTy != MVT::i64)
12099 return Register();
12100 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12101}
12102
12103Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12104 switch (VT.SimpleTy) {
12105 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
12106 default: return Register();
12107 }
12108}
12109
12110// FastEmit functions for ISD::OR.
12111
12112Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12113 if (RetVT.SimpleTy != MVT::i32)
12114 return Register();
12115 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12116}
12117
12118Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12119 if (RetVT.SimpleTy != MVT::i64)
12120 return Register();
12121 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12122}
12123
12124Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12125 if (RetVT.SimpleTy != MVT::v8i8)
12126 return Register();
12127 if ((Subtarget->isNeonAvailable())) {
12128 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12129 }
12130 return Register();
12131}
12132
12133Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12134 if (RetVT.SimpleTy != MVT::v16i8)
12135 return Register();
12136 if ((Subtarget->isNeonAvailable())) {
12137 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12138 }
12139 return Register();
12140}
12141
12142Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12143 if (RetVT.SimpleTy != MVT::v4i16)
12144 return Register();
12145 if ((Subtarget->isNeonAvailable())) {
12146 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12147 }
12148 return Register();
12149}
12150
12151Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12152 if (RetVT.SimpleTy != MVT::v8i16)
12153 return Register();
12154 if ((Subtarget->isNeonAvailable())) {
12155 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12156 }
12157 return Register();
12158}
12159
12160Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12161 if (RetVT.SimpleTy != MVT::v2i32)
12162 return Register();
12163 if ((Subtarget->isNeonAvailable())) {
12164 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12165 }
12166 return Register();
12167}
12168
12169Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12170 if (RetVT.SimpleTy != MVT::v4i32)
12171 return Register();
12172 if ((Subtarget->isNeonAvailable())) {
12173 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12174 }
12175 return Register();
12176}
12177
12178Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12179 if (RetVT.SimpleTy != MVT::v1i64)
12180 return Register();
12181 if ((Subtarget->isNeonAvailable())) {
12182 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12183 }
12184 return Register();
12185}
12186
12187Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12188 if (RetVT.SimpleTy != MVT::v2i64)
12189 return Register();
12190 if ((Subtarget->isNeonAvailable())) {
12191 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12192 }
12193 return Register();
12194}
12195
12196Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12197 if (RetVT.SimpleTy != MVT::nxv16i8)
12198 return Register();
12199 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12200 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12201 }
12202 return Register();
12203}
12204
12205Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12206 if (RetVT.SimpleTy != MVT::nxv8i16)
12207 return Register();
12208 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12209 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12210 }
12211 return Register();
12212}
12213
12214Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12215 if (RetVT.SimpleTy != MVT::nxv4i32)
12216 return Register();
12217 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12218 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12219 }
12220 return Register();
12221}
12222
12223Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12224 if (RetVT.SimpleTy != MVT::nxv2i64)
12225 return Register();
12226 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12227 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12228 }
12229 return Register();
12230}
12231
12232Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12233 switch (VT.SimpleTy) {
12234 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12235 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12236 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12237 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12238 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12239 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12240 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12241 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12242 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12243 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12244 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12245 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12246 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12247 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12248 default: return Register();
12249 }
12250}
12251
12252// FastEmit functions for ISD::ROTR.
12253
12254Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12255 if (RetVT.SimpleTy != MVT::i64)
12256 return Register();
12257 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12258}
12259
12260Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12261 switch (VT.SimpleTy) {
12262 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12263 default: return Register();
12264 }
12265}
12266
12267// FastEmit functions for ISD::SADDSAT.
12268
12269Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12270 if (RetVT.SimpleTy != MVT::v8i8)
12271 return Register();
12272 if ((Subtarget->isNeonAvailable())) {
12273 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12274 }
12275 return Register();
12276}
12277
12278Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12279 if (RetVT.SimpleTy != MVT::v16i8)
12280 return Register();
12281 if ((Subtarget->isNeonAvailable())) {
12282 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12283 }
12284 return Register();
12285}
12286
12287Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12288 if (RetVT.SimpleTy != MVT::v4i16)
12289 return Register();
12290 if ((Subtarget->isNeonAvailable())) {
12291 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12292 }
12293 return Register();
12294}
12295
12296Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12297 if (RetVT.SimpleTy != MVT::v8i16)
12298 return Register();
12299 if ((Subtarget->isNeonAvailable())) {
12300 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12301 }
12302 return Register();
12303}
12304
12305Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12306 if (RetVT.SimpleTy != MVT::v2i32)
12307 return Register();
12308 if ((Subtarget->isNeonAvailable())) {
12309 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12310 }
12311 return Register();
12312}
12313
12314Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12315 if (RetVT.SimpleTy != MVT::v4i32)
12316 return Register();
12317 if ((Subtarget->isNeonAvailable())) {
12318 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12319 }
12320 return Register();
12321}
12322
12323Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12324 if (RetVT.SimpleTy != MVT::v1i64)
12325 return Register();
12326 if ((Subtarget->isNeonAvailable())) {
12327 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12328 }
12329 return Register();
12330}
12331
12332Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12333 if (RetVT.SimpleTy != MVT::v2i64)
12334 return Register();
12335 if ((Subtarget->isNeonAvailable())) {
12336 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12337 }
12338 return Register();
12339}
12340
12341Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12342 if (RetVT.SimpleTy != MVT::nxv16i8)
12343 return Register();
12344 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12345 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12346 }
12347 return Register();
12348}
12349
12350Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12351 if (RetVT.SimpleTy != MVT::nxv8i16)
12352 return Register();
12353 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12354 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12355 }
12356 return Register();
12357}
12358
12359Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12360 if (RetVT.SimpleTy != MVT::nxv4i32)
12361 return Register();
12362 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12363 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12364 }
12365 return Register();
12366}
12367
12368Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12369 if (RetVT.SimpleTy != MVT::nxv2i64)
12370 return Register();
12371 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12372 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12373 }
12374 return Register();
12375}
12376
12377Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12378 switch (VT.SimpleTy) {
12379 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12380 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12381 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12382 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12383 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12384 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12385 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12386 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12387 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12388 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12389 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12390 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12391 default: return Register();
12392 }
12393}
12394
12395// FastEmit functions for ISD::SDIV.
12396
12397Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12398 if (RetVT.SimpleTy != MVT::i32)
12399 return Register();
12400 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12401}
12402
12403Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12404 if (RetVT.SimpleTy != MVT::i64)
12405 return Register();
12406 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12407}
12408
12409Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12410 switch (VT.SimpleTy) {
12411 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12412 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12413 default: return Register();
12414 }
12415}
12416
12417// FastEmit functions for ISD::SHL.
12418
12419Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12420 if (RetVT.SimpleTy != MVT::i64)
12421 return Register();
12422 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12423}
12424
12425Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12426 switch (VT.SimpleTy) {
12427 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12428 default: return Register();
12429 }
12430}
12431
12432// FastEmit functions for ISD::SMAX.
12433
12434Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12435 if (RetVT.SimpleTy != MVT::i32)
12436 return Register();
12437 if ((Subtarget->hasCSSC())) {
12438 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12439 }
12440 return Register();
12441}
12442
12443Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12444 if (RetVT.SimpleTy != MVT::i64)
12445 return Register();
12446 if ((Subtarget->hasCSSC())) {
12447 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12448 }
12449 return Register();
12450}
12451
12452Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12453 if (RetVT.SimpleTy != MVT::v8i8)
12454 return Register();
12455 if ((Subtarget->isNeonAvailable())) {
12456 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12457 }
12458 return Register();
12459}
12460
12461Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12462 if (RetVT.SimpleTy != MVT::v16i8)
12463 return Register();
12464 if ((Subtarget->isNeonAvailable())) {
12465 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12466 }
12467 return Register();
12468}
12469
12470Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12471 if (RetVT.SimpleTy != MVT::v4i16)
12472 return Register();
12473 if ((Subtarget->isNeonAvailable())) {
12474 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12475 }
12476 return Register();
12477}
12478
12479Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12480 if (RetVT.SimpleTy != MVT::v8i16)
12481 return Register();
12482 if ((Subtarget->isNeonAvailable())) {
12483 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12484 }
12485 return Register();
12486}
12487
12488Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12489 if (RetVT.SimpleTy != MVT::v2i32)
12490 return Register();
12491 if ((Subtarget->isNeonAvailable())) {
12492 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12493 }
12494 return Register();
12495}
12496
12497Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12498 if (RetVT.SimpleTy != MVT::v4i32)
12499 return Register();
12500 if ((Subtarget->isNeonAvailable())) {
12501 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12502 }
12503 return Register();
12504}
12505
12506Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12507 switch (VT.SimpleTy) {
12508 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12509 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12510 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12511 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12512 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12513 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12514 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12515 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12516 default: return Register();
12517 }
12518}
12519
12520// FastEmit functions for ISD::SMIN.
12521
12522Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12523 if (RetVT.SimpleTy != MVT::i32)
12524 return Register();
12525 if ((Subtarget->hasCSSC())) {
12526 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12527 }
12528 return Register();
12529}
12530
12531Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12532 if (RetVT.SimpleTy != MVT::i64)
12533 return Register();
12534 if ((Subtarget->hasCSSC())) {
12535 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12536 }
12537 return Register();
12538}
12539
12540Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12541 if (RetVT.SimpleTy != MVT::v8i8)
12542 return Register();
12543 if ((Subtarget->isNeonAvailable())) {
12544 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12545 }
12546 return Register();
12547}
12548
12549Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12550 if (RetVT.SimpleTy != MVT::v16i8)
12551 return Register();
12552 if ((Subtarget->isNeonAvailable())) {
12553 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12554 }
12555 return Register();
12556}
12557
12558Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12559 if (RetVT.SimpleTy != MVT::v4i16)
12560 return Register();
12561 if ((Subtarget->isNeonAvailable())) {
12562 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12563 }
12564 return Register();
12565}
12566
12567Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12568 if (RetVT.SimpleTy != MVT::v8i16)
12569 return Register();
12570 if ((Subtarget->isNeonAvailable())) {
12571 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12572 }
12573 return Register();
12574}
12575
12576Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12577 if (RetVT.SimpleTy != MVT::v2i32)
12578 return Register();
12579 if ((Subtarget->isNeonAvailable())) {
12580 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12581 }
12582 return Register();
12583}
12584
12585Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12586 if (RetVT.SimpleTy != MVT::v4i32)
12587 return Register();
12588 if ((Subtarget->isNeonAvailable())) {
12589 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12590 }
12591 return Register();
12592}
12593
12594Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12595 switch (VT.SimpleTy) {
12596 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12597 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12598 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12599 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12600 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12601 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12602 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12603 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12604 default: return Register();
12605 }
12606}
12607
12608// FastEmit functions for ISD::SRA.
12609
12610Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12611 if (RetVT.SimpleTy != MVT::i64)
12612 return Register();
12613 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12614}
12615
12616Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12617 switch (VT.SimpleTy) {
12618 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12619 default: return Register();
12620 }
12621}
12622
12623// FastEmit functions for ISD::SRL.
12624
12625Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12626 if (RetVT.SimpleTy != MVT::i64)
12627 return Register();
12628 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12629}
12630
12631Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12632 switch (VT.SimpleTy) {
12633 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12634 default: return Register();
12635 }
12636}
12637
12638// FastEmit functions for ISD::SSUBSAT.
12639
12640Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12641 if (RetVT.SimpleTy != MVT::v8i8)
12642 return Register();
12643 if ((Subtarget->isNeonAvailable())) {
12644 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12645 }
12646 return Register();
12647}
12648
12649Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12650 if (RetVT.SimpleTy != MVT::v16i8)
12651 return Register();
12652 if ((Subtarget->isNeonAvailable())) {
12653 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12654 }
12655 return Register();
12656}
12657
12658Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12659 if (RetVT.SimpleTy != MVT::v4i16)
12660 return Register();
12661 if ((Subtarget->isNeonAvailable())) {
12662 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12663 }
12664 return Register();
12665}
12666
12667Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12668 if (RetVT.SimpleTy != MVT::v8i16)
12669 return Register();
12670 if ((Subtarget->isNeonAvailable())) {
12671 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12672 }
12673 return Register();
12674}
12675
12676Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12677 if (RetVT.SimpleTy != MVT::v2i32)
12678 return Register();
12679 if ((Subtarget->isNeonAvailable())) {
12680 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12681 }
12682 return Register();
12683}
12684
12685Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12686 if (RetVT.SimpleTy != MVT::v4i32)
12687 return Register();
12688 if ((Subtarget->isNeonAvailable())) {
12689 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12690 }
12691 return Register();
12692}
12693
12694Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12695 if (RetVT.SimpleTy != MVT::v1i64)
12696 return Register();
12697 if ((Subtarget->isNeonAvailable())) {
12698 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12699 }
12700 return Register();
12701}
12702
12703Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12704 if (RetVT.SimpleTy != MVT::v2i64)
12705 return Register();
12706 if ((Subtarget->isNeonAvailable())) {
12707 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12708 }
12709 return Register();
12710}
12711
12712Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12713 if (RetVT.SimpleTy != MVT::nxv16i8)
12714 return Register();
12715 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12716 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12717 }
12718 return Register();
12719}
12720
12721Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12722 if (RetVT.SimpleTy != MVT::nxv8i16)
12723 return Register();
12724 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12725 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12726 }
12727 return Register();
12728}
12729
12730Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12731 if (RetVT.SimpleTy != MVT::nxv4i32)
12732 return Register();
12733 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12734 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12735 }
12736 return Register();
12737}
12738
12739Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12740 if (RetVT.SimpleTy != MVT::nxv2i64)
12741 return Register();
12742 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12743 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12744 }
12745 return Register();
12746}
12747
12748Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12749 switch (VT.SimpleTy) {
12750 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12751 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12752 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12753 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12754 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12755 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12756 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12757 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12758 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12759 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12760 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12761 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12762 default: return Register();
12763 }
12764}
12765
12766// FastEmit functions for ISD::STRICT_FADD.
12767
12768Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12769 if (RetVT.SimpleTy != MVT::f16)
12770 return Register();
12771 if ((Subtarget->hasFullFP16())) {
12772 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12773 }
12774 return Register();
12775}
12776
12777Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12778 if (RetVT.SimpleTy != MVT::f32)
12779 return Register();
12780 if ((Subtarget->hasFPARMv8())) {
12781 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12782 }
12783 return Register();
12784}
12785
12786Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12787 if (RetVT.SimpleTy != MVT::f64)
12788 return Register();
12789 if ((Subtarget->hasFPARMv8())) {
12790 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12791 }
12792 return Register();
12793}
12794
12795Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12796 if (RetVT.SimpleTy != MVT::v4f16)
12797 return Register();
12798 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12799 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12800 }
12801 return Register();
12802}
12803
12804Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12805 if (RetVT.SimpleTy != MVT::v8f16)
12806 return Register();
12807 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12808 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12809 }
12810 return Register();
12811}
12812
12813Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12814 if (RetVT.SimpleTy != MVT::v2f32)
12815 return Register();
12816 if ((Subtarget->isNeonAvailable())) {
12817 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12818 }
12819 return Register();
12820}
12821
12822Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12823 if (RetVT.SimpleTy != MVT::v4f32)
12824 return Register();
12825 if ((Subtarget->isNeonAvailable())) {
12826 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12827 }
12828 return Register();
12829}
12830
12831Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12832 if (RetVT.SimpleTy != MVT::v2f64)
12833 return Register();
12834 if ((Subtarget->isNeonAvailable())) {
12835 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12836 }
12837 return Register();
12838}
12839
12840Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12841 switch (VT.SimpleTy) {
12842 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12843 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12844 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12845 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12846 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12847 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12848 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12849 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12850 default: return Register();
12851 }
12852}
12853
12854// FastEmit functions for ISD::STRICT_FDIV.
12855
12856Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12857 if (RetVT.SimpleTy != MVT::f16)
12858 return Register();
12859 if ((Subtarget->hasFullFP16())) {
12860 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12861 }
12862 return Register();
12863}
12864
12865Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12866 if (RetVT.SimpleTy != MVT::f32)
12867 return Register();
12868 if ((Subtarget->hasFPARMv8())) {
12869 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12870 }
12871 return Register();
12872}
12873
12874Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12875 if (RetVT.SimpleTy != MVT::f64)
12876 return Register();
12877 if ((Subtarget->hasFPARMv8())) {
12878 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12879 }
12880 return Register();
12881}
12882
12883Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12884 if (RetVT.SimpleTy != MVT::v4f16)
12885 return Register();
12886 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12887 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12888 }
12889 return Register();
12890}
12891
12892Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12893 if (RetVT.SimpleTy != MVT::v8f16)
12894 return Register();
12895 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12896 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12897 }
12898 return Register();
12899}
12900
12901Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12902 if (RetVT.SimpleTy != MVT::v2f32)
12903 return Register();
12904 if ((Subtarget->isNeonAvailable())) {
12905 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12906 }
12907 return Register();
12908}
12909
12910Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12911 if (RetVT.SimpleTy != MVT::v4f32)
12912 return Register();
12913 if ((Subtarget->isNeonAvailable())) {
12914 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12915 }
12916 return Register();
12917}
12918
12919Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12920 if (RetVT.SimpleTy != MVT::v2f64)
12921 return Register();
12922 if ((Subtarget->isNeonAvailable())) {
12923 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12924 }
12925 return Register();
12926}
12927
12928Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12929 switch (VT.SimpleTy) {
12930 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12931 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12932 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12933 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12934 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12935 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12936 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12937 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12938 default: return Register();
12939 }
12940}
12941
12942// FastEmit functions for ISD::STRICT_FMAXIMUM.
12943
12944Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12945 if (RetVT.SimpleTy != MVT::f16)
12946 return Register();
12947 if ((Subtarget->hasFullFP16())) {
12948 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12949 }
12950 return Register();
12951}
12952
12953Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12954 if (RetVT.SimpleTy != MVT::f32)
12955 return Register();
12956 if ((Subtarget->hasFPARMv8())) {
12957 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12958 }
12959 return Register();
12960}
12961
12962Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12963 if (RetVT.SimpleTy != MVT::f64)
12964 return Register();
12965 if ((Subtarget->hasFPARMv8())) {
12966 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12967 }
12968 return Register();
12969}
12970
12971Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12972 if (RetVT.SimpleTy != MVT::v4f16)
12973 return Register();
12974 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12975 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12976 }
12977 return Register();
12978}
12979
12980Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12981 if (RetVT.SimpleTy != MVT::v8f16)
12982 return Register();
12983 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12984 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12985 }
12986 return Register();
12987}
12988
12989Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12990 if (RetVT.SimpleTy != MVT::v2f32)
12991 return Register();
12992 if ((Subtarget->isNeonAvailable())) {
12993 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12994 }
12995 return Register();
12996}
12997
12998Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12999 if (RetVT.SimpleTy != MVT::v4f32)
13000 return Register();
13001 if ((Subtarget->isNeonAvailable())) {
13002 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13003 }
13004 return Register();
13005}
13006
13007Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13008 if (RetVT.SimpleTy != MVT::v2f64)
13009 return Register();
13010 if ((Subtarget->isNeonAvailable())) {
13011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13012 }
13013 return Register();
13014}
13015
13016Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13017 switch (VT.SimpleTy) {
13018 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13019 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13020 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13021 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13022 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13023 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13024 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13025 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13026 default: return Register();
13027 }
13028}
13029
13030// FastEmit functions for ISD::STRICT_FMAXNUM.
13031
13032Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13033 if (RetVT.SimpleTy != MVT::f16)
13034 return Register();
13035 if ((Subtarget->hasFullFP16())) {
13036 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13037 }
13038 return Register();
13039}
13040
13041Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13042 if (RetVT.SimpleTy != MVT::f32)
13043 return Register();
13044 if ((Subtarget->hasFPARMv8())) {
13045 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13046 }
13047 return Register();
13048}
13049
13050Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13051 if (RetVT.SimpleTy != MVT::f64)
13052 return Register();
13053 if ((Subtarget->hasFPARMv8())) {
13054 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13055 }
13056 return Register();
13057}
13058
13059Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13060 if (RetVT.SimpleTy != MVT::v4f16)
13061 return Register();
13062 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13063 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13064 }
13065 return Register();
13066}
13067
13068Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13069 if (RetVT.SimpleTy != MVT::v8f16)
13070 return Register();
13071 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13072 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13073 }
13074 return Register();
13075}
13076
13077Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13078 if (RetVT.SimpleTy != MVT::v2f32)
13079 return Register();
13080 if ((Subtarget->isNeonAvailable())) {
13081 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13082 }
13083 return Register();
13084}
13085
13086Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13087 if (RetVT.SimpleTy != MVT::v4f32)
13088 return Register();
13089 if ((Subtarget->isNeonAvailable())) {
13090 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13091 }
13092 return Register();
13093}
13094
13095Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13096 if (RetVT.SimpleTy != MVT::v2f64)
13097 return Register();
13098 if ((Subtarget->isNeonAvailable())) {
13099 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13100 }
13101 return Register();
13102}
13103
13104Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13105 switch (VT.SimpleTy) {
13106 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
13107 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
13108 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
13109 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13110 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13111 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13112 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13113 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13114 default: return Register();
13115 }
13116}
13117
13118// FastEmit functions for ISD::STRICT_FMINIMUM.
13119
13120Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13121 if (RetVT.SimpleTy != MVT::f16)
13122 return Register();
13123 if ((Subtarget->hasFullFP16())) {
13124 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13125 }
13126 return Register();
13127}
13128
13129Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13130 if (RetVT.SimpleTy != MVT::f32)
13131 return Register();
13132 if ((Subtarget->hasFPARMv8())) {
13133 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13134 }
13135 return Register();
13136}
13137
13138Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13139 if (RetVT.SimpleTy != MVT::f64)
13140 return Register();
13141 if ((Subtarget->hasFPARMv8())) {
13142 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13143 }
13144 return Register();
13145}
13146
13147Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13148 if (RetVT.SimpleTy != MVT::v4f16)
13149 return Register();
13150 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13151 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13152 }
13153 return Register();
13154}
13155
13156Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13157 if (RetVT.SimpleTy != MVT::v8f16)
13158 return Register();
13159 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13160 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13161 }
13162 return Register();
13163}
13164
13165Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13166 if (RetVT.SimpleTy != MVT::v2f32)
13167 return Register();
13168 if ((Subtarget->isNeonAvailable())) {
13169 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13170 }
13171 return Register();
13172}
13173
13174Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13175 if (RetVT.SimpleTy != MVT::v4f32)
13176 return Register();
13177 if ((Subtarget->isNeonAvailable())) {
13178 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13179 }
13180 return Register();
13181}
13182
13183Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13184 if (RetVT.SimpleTy != MVT::v2f64)
13185 return Register();
13186 if ((Subtarget->isNeonAvailable())) {
13187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13188 }
13189 return Register();
13190}
13191
13192Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13193 switch (VT.SimpleTy) {
13194 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13195 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13196 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13197 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13198 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13199 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13200 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13201 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13202 default: return Register();
13203 }
13204}
13205
13206// FastEmit functions for ISD::STRICT_FMINNUM.
13207
13208Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13209 if (RetVT.SimpleTy != MVT::f16)
13210 return Register();
13211 if ((Subtarget->hasFullFP16())) {
13212 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13213 }
13214 return Register();
13215}
13216
13217Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13218 if (RetVT.SimpleTy != MVT::f32)
13219 return Register();
13220 if ((Subtarget->hasFPARMv8())) {
13221 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13222 }
13223 return Register();
13224}
13225
13226Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13227 if (RetVT.SimpleTy != MVT::f64)
13228 return Register();
13229 if ((Subtarget->hasFPARMv8())) {
13230 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13231 }
13232 return Register();
13233}
13234
13235Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13236 if (RetVT.SimpleTy != MVT::v4f16)
13237 return Register();
13238 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13239 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13240 }
13241 return Register();
13242}
13243
13244Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13245 if (RetVT.SimpleTy != MVT::v8f16)
13246 return Register();
13247 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13248 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13249 }
13250 return Register();
13251}
13252
13253Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13254 if (RetVT.SimpleTy != MVT::v2f32)
13255 return Register();
13256 if ((Subtarget->isNeonAvailable())) {
13257 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13258 }
13259 return Register();
13260}
13261
13262Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13263 if (RetVT.SimpleTy != MVT::v4f32)
13264 return Register();
13265 if ((Subtarget->isNeonAvailable())) {
13266 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13267 }
13268 return Register();
13269}
13270
13271Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13272 if (RetVT.SimpleTy != MVT::v2f64)
13273 return Register();
13274 if ((Subtarget->isNeonAvailable())) {
13275 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13276 }
13277 return Register();
13278}
13279
13280Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13281 switch (VT.SimpleTy) {
13282 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13283 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13284 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13285 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13286 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13287 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13288 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13289 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13290 default: return Register();
13291 }
13292}
13293
13294// FastEmit functions for ISD::STRICT_FMUL.
13295
13296Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13297 if (RetVT.SimpleTy != MVT::f16)
13298 return Register();
13299 if ((Subtarget->hasFullFP16())) {
13300 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13301 }
13302 return Register();
13303}
13304
13305Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13306 if (RetVT.SimpleTy != MVT::f32)
13307 return Register();
13308 if ((Subtarget->hasFPARMv8())) {
13309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13310 }
13311 return Register();
13312}
13313
13314Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13315 if (RetVT.SimpleTy != MVT::f64)
13316 return Register();
13317 if ((Subtarget->hasFPARMv8())) {
13318 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13319 }
13320 return Register();
13321}
13322
13323Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13324 if (RetVT.SimpleTy != MVT::v4f16)
13325 return Register();
13326 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13327 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13328 }
13329 return Register();
13330}
13331
13332Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13333 if (RetVT.SimpleTy != MVT::v8f16)
13334 return Register();
13335 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13336 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13337 }
13338 return Register();
13339}
13340
13341Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13342 if (RetVT.SimpleTy != MVT::v2f32)
13343 return Register();
13344 if ((Subtarget->isNeonAvailable())) {
13345 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13346 }
13347 return Register();
13348}
13349
13350Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13351 if (RetVT.SimpleTy != MVT::v4f32)
13352 return Register();
13353 if ((Subtarget->isNeonAvailable())) {
13354 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13355 }
13356 return Register();
13357}
13358
13359Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13360 if (RetVT.SimpleTy != MVT::v2f64)
13361 return Register();
13362 if ((Subtarget->isNeonAvailable())) {
13363 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13364 }
13365 return Register();
13366}
13367
13368Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13369 switch (VT.SimpleTy) {
13370 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13371 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13372 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13373 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13374 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13375 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13376 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13377 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13378 default: return Register();
13379 }
13380}
13381
13382// FastEmit functions for ISD::STRICT_FSUB.
13383
13384Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
13385 if (RetVT.SimpleTy != MVT::f16)
13386 return Register();
13387 if ((Subtarget->hasFullFP16())) {
13388 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13389 }
13390 return Register();
13391}
13392
13393Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
13394 if (RetVT.SimpleTy != MVT::f32)
13395 return Register();
13396 if ((Subtarget->hasFPARMv8())) {
13397 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13398 }
13399 return Register();
13400}
13401
13402Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
13403 if (RetVT.SimpleTy != MVT::f64)
13404 return Register();
13405 if ((Subtarget->hasFPARMv8())) {
13406 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13407 }
13408 return Register();
13409}
13410
13411Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
13412 if (RetVT.SimpleTy != MVT::v4f16)
13413 return Register();
13414 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13415 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13416 }
13417 return Register();
13418}
13419
13420Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
13421 if (RetVT.SimpleTy != MVT::v8f16)
13422 return Register();
13423 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13424 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13425 }
13426 return Register();
13427}
13428
13429Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
13430 if (RetVT.SimpleTy != MVT::v2f32)
13431 return Register();
13432 if ((Subtarget->isNeonAvailable())) {
13433 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13434 }
13435 return Register();
13436}
13437
13438Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13439 if (RetVT.SimpleTy != MVT::v4f32)
13440 return Register();
13441 if ((Subtarget->isNeonAvailable())) {
13442 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13443 }
13444 return Register();
13445}
13446
13447Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13448 if (RetVT.SimpleTy != MVT::v2f64)
13449 return Register();
13450 if ((Subtarget->isNeonAvailable())) {
13451 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13452 }
13453 return Register();
13454}
13455
13456Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13457 switch (VT.SimpleTy) {
13458 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13459 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13460 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13461 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13462 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13463 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13464 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13465 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13466 default: return Register();
13467 }
13468}
13469
13470// FastEmit functions for ISD::SUB.
13471
13472Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13473 if (RetVT.SimpleTy != MVT::i32)
13474 return Register();
13475 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13476}
13477
13478Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13479 if (RetVT.SimpleTy != MVT::i64)
13480 return Register();
13481 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13482}
13483
13484Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13485 if (RetVT.SimpleTy != MVT::v8i8)
13486 return Register();
13487 if ((Subtarget->isNeonAvailable())) {
13488 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13489 }
13490 return Register();
13491}
13492
13493Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13494 if (RetVT.SimpleTy != MVT::v16i8)
13495 return Register();
13496 if ((Subtarget->isNeonAvailable())) {
13497 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13498 }
13499 return Register();
13500}
13501
13502Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13503 if (RetVT.SimpleTy != MVT::v4i16)
13504 return Register();
13505 if ((Subtarget->isNeonAvailable())) {
13506 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13507 }
13508 return Register();
13509}
13510
13511Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13512 if (RetVT.SimpleTy != MVT::v8i16)
13513 return Register();
13514 if ((Subtarget->isNeonAvailable())) {
13515 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13516 }
13517 return Register();
13518}
13519
13520Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13521 if (RetVT.SimpleTy != MVT::v2i32)
13522 return Register();
13523 if ((Subtarget->isNeonAvailable())) {
13524 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13525 }
13526 return Register();
13527}
13528
13529Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13530 if (RetVT.SimpleTy != MVT::v4i32)
13531 return Register();
13532 if ((Subtarget->isNeonAvailable())) {
13533 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13534 }
13535 return Register();
13536}
13537
13538Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13539 if (RetVT.SimpleTy != MVT::v1i64)
13540 return Register();
13541 if ((Subtarget->isNeonAvailable())) {
13542 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13543 }
13544 return Register();
13545}
13546
13547Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13548 if (RetVT.SimpleTy != MVT::v2i64)
13549 return Register();
13550 if ((Subtarget->isNeonAvailable())) {
13551 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13552 }
13553 return Register();
13554}
13555
13556Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13557 if (RetVT.SimpleTy != MVT::nxv16i8)
13558 return Register();
13559 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13560 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13561 }
13562 return Register();
13563}
13564
13565Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13566 if (RetVT.SimpleTy != MVT::nxv8i16)
13567 return Register();
13568 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13569 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13570 }
13571 return Register();
13572}
13573
13574Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13575 if (RetVT.SimpleTy != MVT::nxv4i32)
13576 return Register();
13577 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13578 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13579 }
13580 return Register();
13581}
13582
13583Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13584 if (RetVT.SimpleTy != MVT::nxv2i64)
13585 return Register();
13586 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13587 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13588 }
13589 return Register();
13590}
13591
13592Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13593 switch (VT.SimpleTy) {
13594 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13595 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13596 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13597 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13598 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13599 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13600 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13601 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13602 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13603 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13604 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13605 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13606 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13607 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13608 default: return Register();
13609 }
13610}
13611
13612// FastEmit functions for ISD::UADDSAT.
13613
13614Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13615 if (RetVT.SimpleTy != MVT::v8i8)
13616 return Register();
13617 if ((Subtarget->isNeonAvailable())) {
13618 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13619 }
13620 return Register();
13621}
13622
13623Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13624 if (RetVT.SimpleTy != MVT::v16i8)
13625 return Register();
13626 if ((Subtarget->isNeonAvailable())) {
13627 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13628 }
13629 return Register();
13630}
13631
13632Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13633 if (RetVT.SimpleTy != MVT::v4i16)
13634 return Register();
13635 if ((Subtarget->isNeonAvailable())) {
13636 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13637 }
13638 return Register();
13639}
13640
13641Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13642 if (RetVT.SimpleTy != MVT::v8i16)
13643 return Register();
13644 if ((Subtarget->isNeonAvailable())) {
13645 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13646 }
13647 return Register();
13648}
13649
13650Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13651 if (RetVT.SimpleTy != MVT::v2i32)
13652 return Register();
13653 if ((Subtarget->isNeonAvailable())) {
13654 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13655 }
13656 return Register();
13657}
13658
13659Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13660 if (RetVT.SimpleTy != MVT::v4i32)
13661 return Register();
13662 if ((Subtarget->isNeonAvailable())) {
13663 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13664 }
13665 return Register();
13666}
13667
13668Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13669 if (RetVT.SimpleTy != MVT::v1i64)
13670 return Register();
13671 if ((Subtarget->isNeonAvailable())) {
13672 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13673 }
13674 return Register();
13675}
13676
13677Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13678 if (RetVT.SimpleTy != MVT::v2i64)
13679 return Register();
13680 if ((Subtarget->isNeonAvailable())) {
13681 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13682 }
13683 return Register();
13684}
13685
13686Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13687 if (RetVT.SimpleTy != MVT::nxv16i8)
13688 return Register();
13689 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13690 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13691 }
13692 return Register();
13693}
13694
13695Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13696 if (RetVT.SimpleTy != MVT::nxv8i16)
13697 return Register();
13698 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13699 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13700 }
13701 return Register();
13702}
13703
13704Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13705 if (RetVT.SimpleTy != MVT::nxv4i32)
13706 return Register();
13707 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13708 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13709 }
13710 return Register();
13711}
13712
13713Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13714 if (RetVT.SimpleTy != MVT::nxv2i64)
13715 return Register();
13716 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13717 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13718 }
13719 return Register();
13720}
13721
13722Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13723 switch (VT.SimpleTy) {
13724 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13725 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13726 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13727 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13728 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13729 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13730 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13731 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13732 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13733 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13734 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13735 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13736 default: return Register();
13737 }
13738}
13739
13740// FastEmit functions for ISD::UDIV.
13741
13742Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13743 if (RetVT.SimpleTy != MVT::i32)
13744 return Register();
13745 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13746}
13747
13748Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13749 if (RetVT.SimpleTy != MVT::i64)
13750 return Register();
13751 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13752}
13753
13754Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13755 switch (VT.SimpleTy) {
13756 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13757 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13758 default: return Register();
13759 }
13760}
13761
13762// FastEmit functions for ISD::UMAX.
13763
13764Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13765 if (RetVT.SimpleTy != MVT::i32)
13766 return Register();
13767 if ((Subtarget->hasCSSC())) {
13768 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13769 }
13770 return Register();
13771}
13772
13773Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13774 if (RetVT.SimpleTy != MVT::i64)
13775 return Register();
13776 if ((Subtarget->hasCSSC())) {
13777 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13778 }
13779 return Register();
13780}
13781
13782Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13783 if (RetVT.SimpleTy != MVT::v8i8)
13784 return Register();
13785 if ((Subtarget->isNeonAvailable())) {
13786 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13787 }
13788 return Register();
13789}
13790
13791Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13792 if (RetVT.SimpleTy != MVT::v16i8)
13793 return Register();
13794 if ((Subtarget->isNeonAvailable())) {
13795 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13796 }
13797 return Register();
13798}
13799
13800Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13801 if (RetVT.SimpleTy != MVT::v4i16)
13802 return Register();
13803 if ((Subtarget->isNeonAvailable())) {
13804 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13805 }
13806 return Register();
13807}
13808
13809Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13810 if (RetVT.SimpleTy != MVT::v8i16)
13811 return Register();
13812 if ((Subtarget->isNeonAvailable())) {
13813 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13814 }
13815 return Register();
13816}
13817
13818Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13819 if (RetVT.SimpleTy != MVT::v2i32)
13820 return Register();
13821 if ((Subtarget->isNeonAvailable())) {
13822 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13823 }
13824 return Register();
13825}
13826
13827Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13828 if (RetVT.SimpleTy != MVT::v4i32)
13829 return Register();
13830 if ((Subtarget->isNeonAvailable())) {
13831 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13832 }
13833 return Register();
13834}
13835
13836Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13837 switch (VT.SimpleTy) {
13838 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13839 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13840 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13841 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13842 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13843 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13844 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13845 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13846 default: return Register();
13847 }
13848}
13849
13850// FastEmit functions for ISD::UMIN.
13851
13852Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13853 if (RetVT.SimpleTy != MVT::i32)
13854 return Register();
13855 if ((Subtarget->hasCSSC())) {
13856 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13857 }
13858 return Register();
13859}
13860
13861Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13862 if (RetVT.SimpleTy != MVT::i64)
13863 return Register();
13864 if ((Subtarget->hasCSSC())) {
13865 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13866 }
13867 return Register();
13868}
13869
13870Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13871 if (RetVT.SimpleTy != MVT::v8i8)
13872 return Register();
13873 if ((Subtarget->isNeonAvailable())) {
13874 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13875 }
13876 return Register();
13877}
13878
13879Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13880 if (RetVT.SimpleTy != MVT::v16i8)
13881 return Register();
13882 if ((Subtarget->isNeonAvailable())) {
13883 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13884 }
13885 return Register();
13886}
13887
13888Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13889 if (RetVT.SimpleTy != MVT::v4i16)
13890 return Register();
13891 if ((Subtarget->isNeonAvailable())) {
13892 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13893 }
13894 return Register();
13895}
13896
13897Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13898 if (RetVT.SimpleTy != MVT::v8i16)
13899 return Register();
13900 if ((Subtarget->isNeonAvailable())) {
13901 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13902 }
13903 return Register();
13904}
13905
13906Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13907 if (RetVT.SimpleTy != MVT::v2i32)
13908 return Register();
13909 if ((Subtarget->isNeonAvailable())) {
13910 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13911 }
13912 return Register();
13913}
13914
13915Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13916 if (RetVT.SimpleTy != MVT::v4i32)
13917 return Register();
13918 if ((Subtarget->isNeonAvailable())) {
13919 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13920 }
13921 return Register();
13922}
13923
13924Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13925 switch (VT.SimpleTy) {
13926 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13927 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13928 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13929 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13930 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13931 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13932 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13933 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13934 default: return Register();
13935 }
13936}
13937
13938// FastEmit functions for ISD::USUBSAT.
13939
13940Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13941 if (RetVT.SimpleTy != MVT::v8i8)
13942 return Register();
13943 if ((Subtarget->isNeonAvailable())) {
13944 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13945 }
13946 return Register();
13947}
13948
13949Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13950 if (RetVT.SimpleTy != MVT::v16i8)
13951 return Register();
13952 if ((Subtarget->isNeonAvailable())) {
13953 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13954 }
13955 return Register();
13956}
13957
13958Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13959 if (RetVT.SimpleTy != MVT::v4i16)
13960 return Register();
13961 if ((Subtarget->isNeonAvailable())) {
13962 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13963 }
13964 return Register();
13965}
13966
13967Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13968 if (RetVT.SimpleTy != MVT::v8i16)
13969 return Register();
13970 if ((Subtarget->isNeonAvailable())) {
13971 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13972 }
13973 return Register();
13974}
13975
13976Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13977 if (RetVT.SimpleTy != MVT::v2i32)
13978 return Register();
13979 if ((Subtarget->isNeonAvailable())) {
13980 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13981 }
13982 return Register();
13983}
13984
13985Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13986 if (RetVT.SimpleTy != MVT::v4i32)
13987 return Register();
13988 if ((Subtarget->isNeonAvailable())) {
13989 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13990 }
13991 return Register();
13992}
13993
13994Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13995 if (RetVT.SimpleTy != MVT::v1i64)
13996 return Register();
13997 if ((Subtarget->isNeonAvailable())) {
13998 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13999 }
14000 return Register();
14001}
14002
14003Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14004 if (RetVT.SimpleTy != MVT::v2i64)
14005 return Register();
14006 if ((Subtarget->isNeonAvailable())) {
14007 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
14008 }
14009 return Register();
14010}
14011
14012Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14013 if (RetVT.SimpleTy != MVT::nxv16i8)
14014 return Register();
14015 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14016 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
14017 }
14018 return Register();
14019}
14020
14021Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14022 if (RetVT.SimpleTy != MVT::nxv8i16)
14023 return Register();
14024 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14025 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
14026 }
14027 return Register();
14028}
14029
14030Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14031 if (RetVT.SimpleTy != MVT::nxv4i32)
14032 return Register();
14033 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14034 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
14035 }
14036 return Register();
14037}
14038
14039Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14040 if (RetVT.SimpleTy != MVT::nxv2i64)
14041 return Register();
14042 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14043 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
14044 }
14045 return Register();
14046}
14047
14048Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14049 switch (VT.SimpleTy) {
14050 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
14051 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
14052 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
14053 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
14054 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
14055 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
14056 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
14057 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
14058 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14059 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14060 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14061 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14062 default: return Register();
14063 }
14064}
14065
14066// FastEmit functions for ISD::XOR.
14067
14068Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
14069 if (RetVT.SimpleTy != MVT::i32)
14070 return Register();
14071 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
14072}
14073
14074Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
14075 if (RetVT.SimpleTy != MVT::i64)
14076 return Register();
14077 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
14078}
14079
14080Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
14081 if (RetVT.SimpleTy != MVT::v8i8)
14082 return Register();
14083 if ((Subtarget->isNeonAvailable())) {
14084 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14085 }
14086 return Register();
14087}
14088
14089Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14090 if (RetVT.SimpleTy != MVT::v16i8)
14091 return Register();
14092 if ((Subtarget->isNeonAvailable())) {
14093 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14094 }
14095 return Register();
14096}
14097
14098Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
14099 if (RetVT.SimpleTy != MVT::v4i16)
14100 return Register();
14101 if ((Subtarget->isNeonAvailable())) {
14102 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14103 }
14104 return Register();
14105}
14106
14107Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14108 if (RetVT.SimpleTy != MVT::v8i16)
14109 return Register();
14110 if ((Subtarget->isNeonAvailable())) {
14111 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14112 }
14113 return Register();
14114}
14115
14116Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
14117 if (RetVT.SimpleTy != MVT::v2i32)
14118 return Register();
14119 if ((Subtarget->isNeonAvailable())) {
14120 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14121 }
14122 return Register();
14123}
14124
14125Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14126 if (RetVT.SimpleTy != MVT::v4i32)
14127 return Register();
14128 if ((Subtarget->isNeonAvailable())) {
14129 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14130 }
14131 return Register();
14132}
14133
14134Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
14135 if (RetVT.SimpleTy != MVT::v1i64)
14136 return Register();
14137 if ((Subtarget->isNeonAvailable())) {
14138 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14139 }
14140 return Register();
14141}
14142
14143Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14144 if (RetVT.SimpleTy != MVT::v2i64)
14145 return Register();
14146 if ((Subtarget->isNeonAvailable())) {
14147 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14148 }
14149 return Register();
14150}
14151
14152Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
14153 if (RetVT.SimpleTy != MVT::nxv16i8)
14154 return Register();
14155 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14156 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14157 }
14158 return Register();
14159}
14160
14161Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
14162 if (RetVT.SimpleTy != MVT::nxv8i16)
14163 return Register();
14164 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14165 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14166 }
14167 return Register();
14168}
14169
14170Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
14171 if (RetVT.SimpleTy != MVT::nxv4i32)
14172 return Register();
14173 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14174 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14175 }
14176 return Register();
14177}
14178
14179Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
14180 if (RetVT.SimpleTy != MVT::nxv2i64)
14181 return Register();
14182 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
14183 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14184 }
14185 return Register();
14186}
14187
14188Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
14189 switch (VT.SimpleTy) {
14190 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14191 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14192 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14193 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14194 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14195 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14196 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14197 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14198 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14199 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14200 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14201 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14202 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14203 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14204 default: return Register();
14205 }
14206}
14207
14208// Top-level FastEmit function.
14209
14210Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
14211 switch (Opcode) {
14212 case AArch64ISD::ADDHN: return fastEmit_AArch64ISD_ADDHN_rr(VT, RetVT, Op0, Op1);
14213 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14214 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14215 case AArch64ISD::CMTST: return fastEmit_AArch64ISD_CMTST_rr(VT, RetVT, Op0, Op1);
14216 case AArch64ISD::CTTZ_ELTS_ZERO_POISON: return fastEmit_AArch64ISD_CTTZ_ELTS_ZERO_POISON_rr(VT, RetVT, Op0, Op1);
14217 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14218 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14219 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14220 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14221 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14222 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14223 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14224 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14225 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14226 case AArch64ISD::PTEST_FIRST: return fastEmit_AArch64ISD_PTEST_FIRST_rr(VT, RetVT, Op0, Op1);
14227 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14228 case AArch64ISD::SQADD: return fastEmit_AArch64ISD_SQADD_rr(VT, RetVT, Op0, Op1);
14229 case AArch64ISD::SQDMULH: return fastEmit_AArch64ISD_SQDMULH_rr(VT, RetVT, Op0, Op1);
14230 case AArch64ISD::SQDMULL: return fastEmit_AArch64ISD_SQDMULL_rr(VT, RetVT, Op0, Op1);
14231 case AArch64ISD::SQRDMULH: return fastEmit_AArch64ISD_SQRDMULH_rr(VT, RetVT, Op0, Op1);
14232 case AArch64ISD::SQRSHL: return fastEmit_AArch64ISD_SQRSHL_rr(VT, RetVT, Op0, Op1);
14233 case AArch64ISD::SQSHL: return fastEmit_AArch64ISD_SQSHL_rr(VT, RetVT, Op0, Op1);
14234 case AArch64ISD::SQSUB: return fastEmit_AArch64ISD_SQSUB_rr(VT, RetVT, Op0, Op1);
14235 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14236 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14237 case AArch64ISD::SUQADD: return fastEmit_AArch64ISD_SUQADD_rr(VT, RetVT, Op0, Op1);
14238 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14239 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14240 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14241 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14242 case AArch64ISD::UQADD: return fastEmit_AArch64ISD_UQADD_rr(VT, RetVT, Op0, Op1);
14243 case AArch64ISD::UQRSHL: return fastEmit_AArch64ISD_UQRSHL_rr(VT, RetVT, Op0, Op1);
14244 case AArch64ISD::UQSHL: return fastEmit_AArch64ISD_UQSHL_rr(VT, RetVT, Op0, Op1);
14245 case AArch64ISD::UQSUB: return fastEmit_AArch64ISD_UQSUB_rr(VT, RetVT, Op0, Op1);
14246 case AArch64ISD::USQADD: return fastEmit_AArch64ISD_USQADD_rr(VT, RetVT, Op0, Op1);
14247 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14248 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14249 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14250 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14251 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14252 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14253 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14254 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14255 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14256 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14257 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14258 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14259 case ISD::CLMUL: return fastEmit_ISD_CLMUL_rr(VT, RetVT, Op0, Op1);
14260 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14261 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14262 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14263 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14264 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14265 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14266 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14267 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14268 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
14269 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14270 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14271 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
14272 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14273 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14274 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14275 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14276 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14277 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14278 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14279 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14280 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14281 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14282 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14283 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14284 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14285 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14286 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14287 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14288 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14289 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14290 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14291 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14292 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14293 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14294 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14295 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14296 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14297 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14298 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14299 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14300 default: return Register();
14301 }
14302}
14303
14304// FastEmit functions for AArch64ISD::DUPLANE64.
14305
14306Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14307 if (RetVT.SimpleTy != MVT::v2i64)
14308 return Register();
14309 if ((Subtarget->isNeonAvailable())) {
14310 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14311 }
14312 return Register();
14313}
14314
14315Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14316 if (RetVT.SimpleTy != MVT::v2f64)
14317 return Register();
14318 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14319}
14320
14321Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14322 switch (VT.SimpleTy) {
14323 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14324 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14325 default: return Register();
14326 }
14327}
14328
14329// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14330
14331Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14332 if (RetVT.SimpleTy != MVT::i64)
14333 return Register();
14334 if ((Subtarget->isNeonAvailable())) {
14335 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14336 }
14337 return Register();
14338}
14339
14340Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
14341 if (RetVT.SimpleTy != MVT::f64)
14342 return Register();
14343 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14344}
14345
14346Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14347 switch (VT.SimpleTy) {
14348 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14349 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14350 default: return Register();
14351 }
14352}
14353
14354// Top-level FastEmit function.
14355
14356Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14357 switch (Opcode) {
14358 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14359 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14360 default: return Register();
14361 }
14362}
14363
14364// FastEmit functions for AArch64ISD::DUPLANE32.
14365
14366Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14367 if ((Subtarget->isNeonAvailable())) {
14368 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14369 }
14370 return Register();
14371}
14372
14373Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14374 if ((Subtarget->isNeonAvailable())) {
14375 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14376 }
14377 return Register();
14378}
14379
14380Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14381switch (RetVT.SimpleTy) {
14382 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14383 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14384 default: return Register();
14385}
14386}
14387
14388Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14389 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14390}
14391
14392Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
14393 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14394}
14395
14396Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14397switch (RetVT.SimpleTy) {
14398 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14399 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14400 default: return Register();
14401}
14402}
14403
14404Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14405 switch (VT.SimpleTy) {
14406 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14407 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14408 default: return Register();
14409 }
14410}
14411
14412// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14413
14414Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14415 if (RetVT.SimpleTy != MVT::i32)
14416 return Register();
14417 if ((Subtarget->isNeonAvailable())) {
14418 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14419 }
14420 return Register();
14421}
14422
14423Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
14424 if (RetVT.SimpleTy != MVT::f32)
14425 return Register();
14426 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14427}
14428
14429Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14430 switch (VT.SimpleTy) {
14431 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14432 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14433 default: return Register();
14434 }
14435}
14436
14437// Top-level FastEmit function.
14438
14439Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14440 switch (Opcode) {
14441 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14442 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14443 default: return Register();
14444 }
14445}
14446
14447// FastEmit functions for AArch64ISD::DUPLANE16.
14448
14449Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14450 if ((Subtarget->isNeonAvailable())) {
14451 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14452 }
14453 return Register();
14454}
14455
14456Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14457 if ((Subtarget->isNeonAvailable())) {
14458 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14459 }
14460 return Register();
14461}
14462
14463Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14464switch (RetVT.SimpleTy) {
14465 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14466 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14467 default: return Register();
14468}
14469}
14470
14471Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14472 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14473}
14474
14475Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14476 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14477}
14478
14479Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14480switch (RetVT.SimpleTy) {
14481 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14482 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14483 default: return Register();
14484}
14485}
14486
14487Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14488 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14489}
14490
14491Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14492 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14493}
14494
14495Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14496switch (RetVT.SimpleTy) {
14497 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14498 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14499 default: return Register();
14500}
14501}
14502
14503Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14504 switch (VT.SimpleTy) {
14505 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14506 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14507 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14508 default: return Register();
14509 }
14510}
14511
14512// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14513
14514Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14515 if (RetVT.SimpleTy != MVT::i32)
14516 return Register();
14517 if ((Subtarget->isNeonAvailable())) {
14518 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14519 }
14520 return Register();
14521}
14522
14523Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14524 if (RetVT.SimpleTy != MVT::f16)
14525 return Register();
14526 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14527}
14528
14529Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14530 if (RetVT.SimpleTy != MVT::bf16)
14531 return Register();
14532 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14533}
14534
14535Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14536 switch (VT.SimpleTy) {
14537 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14538 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14539 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14540 default: return Register();
14541 }
14542}
14543
14544// Top-level FastEmit function.
14545
14546Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14547 switch (Opcode) {
14548 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14549 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14550 default: return Register();
14551 }
14552}
14553
14554// FastEmit functions for AArch64ISD::DUPLANE8.
14555
14556Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14557 if ((Subtarget->isNeonAvailable())) {
14558 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14559 }
14560 return Register();
14561}
14562
14563Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14564 if ((Subtarget->isNeonAvailable())) {
14565 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14566 }
14567 return Register();
14568}
14569
14570Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14571switch (RetVT.SimpleTy) {
14572 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14573 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14574 default: return Register();
14575}
14576}
14577
14578Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14579 switch (VT.SimpleTy) {
14580 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14581 default: return Register();
14582 }
14583}
14584
14585// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14586
14587Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14588 if (RetVT.SimpleTy != MVT::i32)
14589 return Register();
14590 if ((Subtarget->isNeonAvailable())) {
14591 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14592 }
14593 return Register();
14594}
14595
14596Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14597 switch (VT.SimpleTy) {
14598 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14599 default: return Register();
14600 }
14601}
14602
14603// Top-level FastEmit function.
14604
14605Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14606 switch (Opcode) {
14607 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14608 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14609 default: return Register();
14610 }
14611}
14612
14613// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14614
14615Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14616 if (RetVT.SimpleTy != MVT::i32)
14617 return Register();
14618 if ((Subtarget->hasNEON())) {
14619 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14620 }
14621 return Register();
14622}
14623
14624Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14625 if (RetVT.SimpleTy != MVT::i32)
14626 return Register();
14627 if ((Subtarget->hasNEON())) {
14628 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14629 }
14630 return Register();
14631}
14632
14633Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14634 if (RetVT.SimpleTy != MVT::i32)
14635 return Register();
14636 if ((Subtarget->hasNEON())) {
14637 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14638 }
14639 return Register();
14640}
14641
14642Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14643 if (RetVT.SimpleTy != MVT::i64)
14644 return Register();
14645 if ((Subtarget->hasNEON())) {
14646 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14647 }
14648 return Register();
14649}
14650
14651Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14652 switch (VT.SimpleTy) {
14653 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14654 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14655 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14656 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14657 default: return Register();
14658 }
14659}
14660
14661// Top-level FastEmit function.
14662
14663Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14664 switch (Opcode) {
14665 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14666 default: return Register();
14667 }
14668}
14669
14670// FastEmit functions for ISD::SMAX.
14671
14672Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14673 if (RetVT.SimpleTy != MVT::i32)
14674 return Register();
14675 if ((Subtarget->hasCSSC())) {
14676 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14677 }
14678 return Register();
14679}
14680
14681Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14682 switch (VT.SimpleTy) {
14683 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14684 default: return Register();
14685 }
14686}
14687
14688// FastEmit functions for ISD::SMIN.
14689
14690Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14691 if (RetVT.SimpleTy != MVT::i32)
14692 return Register();
14693 if ((Subtarget->hasCSSC())) {
14694 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14695 }
14696 return Register();
14697}
14698
14699Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14700 switch (VT.SimpleTy) {
14701 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
14702 default: return Register();
14703 }
14704}
14705
14706// Top-level FastEmit function.
14707
14708Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14709 switch (Opcode) {
14710 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14711 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
14712 default: return Register();
14713 }
14714}
14715
14716// FastEmit functions for ISD::SMAX.
14717
14718Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14719 if (RetVT.SimpleTy != MVT::i64)
14720 return Register();
14721 if ((Subtarget->hasCSSC())) {
14722 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14723 }
14724 return Register();
14725}
14726
14727Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14728 switch (VT.SimpleTy) {
14729 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14730 default: return Register();
14731 }
14732}
14733
14734// FastEmit functions for ISD::SMIN.
14735
14736Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14737 if (RetVT.SimpleTy != MVT::i64)
14738 return Register();
14739 if ((Subtarget->hasCSSC())) {
14740 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14741 }
14742 return Register();
14743}
14744
14745Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14746 switch (VT.SimpleTy) {
14747 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
14748 default: return Register();
14749 }
14750}
14751
14752// Top-level FastEmit function.
14753
14754Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14755 switch (Opcode) {
14756 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14757 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
14758 default: return Register();
14759 }
14760}
14761
14762// FastEmit functions for ISD::UMAX.
14763
14764Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14765 if (RetVT.SimpleTy != MVT::i32)
14766 return Register();
14767 if ((Subtarget->hasCSSC())) {
14768 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14769 }
14770 return Register();
14771}
14772
14773Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14774 switch (VT.SimpleTy) {
14775 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14776 default: return Register();
14777 }
14778}
14779
14780// FastEmit functions for ISD::UMIN.
14781
14782Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
14783 if (RetVT.SimpleTy != MVT::i32)
14784 return Register();
14785 if ((Subtarget->hasCSSC())) {
14786 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14787 }
14788 return Register();
14789}
14790
14791Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14792 switch (VT.SimpleTy) {
14793 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
14794 default: return Register();
14795 }
14796}
14797
14798// Top-level FastEmit function.
14799
14800Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14801 switch (Opcode) {
14802 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14803 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
14804 default: return Register();
14805 }
14806}
14807
14808// FastEmit functions for ISD::UMAX.
14809
14810Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14811 if (RetVT.SimpleTy != MVT::i64)
14812 return Register();
14813 if ((Subtarget->hasCSSC())) {
14814 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14815 }
14816 return Register();
14817}
14818
14819Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14820 switch (VT.SimpleTy) {
14821 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14822 default: return Register();
14823 }
14824}
14825
14826// FastEmit functions for ISD::UMIN.
14827
14828Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
14829 if (RetVT.SimpleTy != MVT::i64)
14830 return Register();
14831 if ((Subtarget->hasCSSC())) {
14832 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14833 }
14834 return Register();
14835}
14836
14837Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14838 switch (VT.SimpleTy) {
14839 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
14840 default: return Register();
14841 }
14842}
14843
14844// Top-level FastEmit function.
14845
14846Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14847 switch (Opcode) {
14848 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14849 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
14850 default: return Register();
14851 }
14852}
14853
14854// FastEmit functions for ISD::Constant.
14855
14856Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
14857 if (RetVT.SimpleTy != MVT::i32)
14858 return Register();
14859 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
14860}
14861
14862Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
14863 if (RetVT.SimpleTy != MVT::i64)
14864 return Register();
14865 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
14866}
14867
14868Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
14869 switch (VT.SimpleTy) {
14870 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
14871 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
14872 default: return Register();
14873 }
14874}
14875
14876// Top-level FastEmit function.
14877
14878Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
14879 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
14880 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
14881 return Reg;
14882
14883 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
14884 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
14885 return Reg;
14886
14887 switch (Opcode) {
14888 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
14889 default: return Register();
14890 }
14891}
14892
14893// FastEmit functions for AArch64ISD::FMOV.
14894
14895Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
14896 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14897 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14898 }
14899 return Register();
14900}
14901
14902Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
14903 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
14904 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14905 }
14906 return Register();
14907}
14908
14909Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
14910 if ((Subtarget->isNeonAvailable())) {
14911 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14912 }
14913 return Register();
14914}
14915
14916Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
14917 if ((Subtarget->isNeonAvailable())) {
14918 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14919 }
14920 return Register();
14921}
14922
14923Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
14924 if ((Subtarget->isNeonAvailable())) {
14925 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14926 }
14927 return Register();
14928}
14929
14930Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14931switch (RetVT.SimpleTy) {
14932 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
14933 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
14934 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
14935 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
14936 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
14937 default: return Register();
14938}
14939}
14940
14941Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14942 switch (VT.SimpleTy) {
14943 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14944 default: return Register();
14945 }
14946}
14947
14948// FastEmit functions for AArch64ISD::MOVI.
14949
14950Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
14951 if ((Subtarget->isNeonAvailable())) {
14952 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
14953 }
14954 return Register();
14955}
14956
14957Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
14958 if ((Subtarget->isNeonAvailable())) {
14959 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14960 }
14961 return Register();
14962}
14963
14964Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14965switch (RetVT.SimpleTy) {
14966 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
14967 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
14968 default: return Register();
14969}
14970}
14971
14972Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
14973 switch (VT.SimpleTy) {
14974 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
14975 default: return Register();
14976 }
14977}
14978
14979// FastEmit functions for AArch64ISD::MOVIedit.
14980
14981Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
14982 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
14983}
14984
14985Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
14986 if ((Subtarget->isNeonAvailable())) {
14987 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
14988 }
14989 return Register();
14990}
14991
14992Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
14993switch (RetVT.SimpleTy) {
14994 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
14995 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
14996 default: return Register();
14997}
14998}
14999
15000Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15001 switch (VT.SimpleTy) {
15002 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15003 default: return Register();
15004 }
15005}
15006
15007// Top-level FastEmit function.
15008
15009Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15010 switch (Opcode) {
15011 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
15012 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
15013 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
15014 default: return Register();
15015 }
15016}
15017
15018// FastEmit functions for AArch64ISD::RDSVL.
15019
15020Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
15021 if (RetVT.SimpleTy != MVT::i64)
15022 return Register();
15023 if ((Subtarget->hasSME())) {
15024 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
15025 }
15026 return Register();
15027}
15028
15029Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
15030 switch (VT.SimpleTy) {
15031 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
15032 default: return Register();
15033 }
15034}
15035
15036// Top-level FastEmit function.
15037
15038Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15039 switch (Opcode) {
15040 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
15041 default: return Register();
15042 }
15043}
15044
15045