1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_vecshiftL64(int64_t Imm) {
57
58 return (((uint32_t)Imm) < 64);
59
60}
61static bool Predicate_vecshiftL32(int64_t Imm) {
62
63 return (((uint32_t)Imm) < 32);
64
65}
66static bool Predicate_vecshiftR64(int64_t Imm) {
67
68 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 65);
69
70}
71static bool Predicate_vecshiftL8(int64_t Imm) {
72
73 return (((uint32_t)Imm) < 8);
74
75}
76static bool Predicate_vecshiftL16(int64_t Imm) {
77
78 return (((uint32_t)Imm) < 16);
79
80}
81static bool Predicate_vecshiftR8(int64_t Imm) {
82
83 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 9);
84
85}
86static bool Predicate_vecshiftR16(int64_t Imm) {
87
88 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
89
90}
91static bool Predicate_vecshiftR32(int64_t Imm) {
92
93 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 33);
94
95}
96static bool Predicate_simm8_32b(int64_t Imm) {
97 return Imm >= -128 && Imm < 128;
98}
99static bool Predicate_simm8_64b(int64_t Imm) {
100 return Imm >= -128 && Imm < 128;
101}
102static bool Predicate_uimm8_32b(int64_t Imm) {
103 return Imm >= 0 && Imm < 256;
104}
105static bool Predicate_uimm8_64b(int64_t Imm) {
106 return Imm >= 0 && Imm < 256;
107}
108static bool Predicate_simm6_32b(int64_t Imm) {
109 return Imm >= -32 && Imm < 32;
110}
111
112
113// FastEmit functions for AArch64ISD::GET_SME_SAVE_SIZE.
114
115Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(MVT RetVT) {
116 if (RetVT.SimpleTy != MVT::i64)
117 return Register();
118 return fastEmitInst_(MachineInstOpcode: AArch64::GetSMESaveSize, RC: &AArch64::GPR64RegClass);
119}
120
121Register fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(MVT VT, MVT RetVT) {
122 switch (VT.SimpleTy) {
123 case MVT::i64: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_MVT_i64_(RetVT);
124 default: return Register();
125 }
126}
127
128// FastEmit functions for AArch64ISD::THREAD_POINTER.
129
130Register fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
131 if (RetVT.SimpleTy != MVT::i64)
132 return Register();
133 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
134}
135
136Register fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
137 switch (VT.SimpleTy) {
138 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
139 default: return Register();
140 }
141}
142
143// Top-level FastEmit function.
144
145Register fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
146 switch (Opcode) {
147 case AArch64ISD::GET_SME_SAVE_SIZE: return fastEmit_AArch64ISD_GET_SME_SAVE_SIZE_(VT, RetVT);
148 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
149 default: return Register();
150 }
151}
152
153// FastEmit functions for AArch64ISD::ALLOCATE_ZA_BUFFER.
154
155Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
156 if (RetVT.SimpleTy != MVT::i64)
157 return Register();
158 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateZABuffer, RC: &AArch64::GPR64spRegClass, Op0);
159}
160
161Register fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
162 switch (VT.SimpleTy) {
163 case MVT::i64: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(RetVT, Op0);
164 default: return Register();
165 }
166}
167
168// FastEmit functions for AArch64ISD::ALLOC_SME_SAVE_BUFFER.
169
170Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(MVT RetVT, Register Op0) {
171 if (RetVT.SimpleTy != MVT::i64)
172 return Register();
173 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateSMESaveBuffer, RC: &AArch64::GPR64spRegClass, Op0);
174}
175
176Register fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(MVT VT, MVT RetVT, Register Op0) {
177 switch (VT.SimpleTy) {
178 case MVT::i64: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_MVT_i64_r(RetVT, Op0);
179 default: return Register();
180 }
181}
182
183// FastEmit functions for AArch64ISD::CALL.
184
185Register fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, Register Op0) {
186 if (RetVT.SimpleTy != MVT::isVoid)
187 return Register();
188 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
189 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
190 }
191 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
192 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
193 }
194 return Register();
195}
196
197Register fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, Register Op0) {
198 switch (VT.SimpleTy) {
199 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
200 default: return Register();
201 }
202}
203
204// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
205
206Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, Register Op0) {
207 if (RetVT.SimpleTy != MVT::bf16)
208 return Register();
209 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
210}
211
212Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, Register Op0) {
213 if (RetVT.SimpleTy != MVT::f16)
214 return Register();
215 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
216}
217
218Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, Register Op0) {
219 if (RetVT.SimpleTy != MVT::f32)
220 return Register();
221 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
222}
223
224Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, Register Op0) {
225 if (RetVT.SimpleTy != MVT::f64)
226 return Register();
227 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
228}
229
230Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, Register Op0) {
231 if (RetVT.SimpleTy != MVT::f128)
232 return Register();
233 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
234}
235
236Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, Register Op0) {
237 if (RetVT.SimpleTy != MVT::v8i8)
238 return Register();
239 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
240}
241
242Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, Register Op0) {
243 if (RetVT.SimpleTy != MVT::v16i8)
244 return Register();
245 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
246}
247
248Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, Register Op0) {
249 if (RetVT.SimpleTy != MVT::v4i16)
250 return Register();
251 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
252}
253
254Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, Register Op0) {
255 if (RetVT.SimpleTy != MVT::v8i16)
256 return Register();
257 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
258}
259
260Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, Register Op0) {
261 if (RetVT.SimpleTy != MVT::v2i32)
262 return Register();
263 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
264}
265
266Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, Register Op0) {
267 if (RetVT.SimpleTy != MVT::v4i32)
268 return Register();
269 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
270}
271
272Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, Register Op0) {
273 if (RetVT.SimpleTy != MVT::v1i64)
274 return Register();
275 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
276}
277
278Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, Register Op0) {
279 if (RetVT.SimpleTy != MVT::v2i64)
280 return Register();
281 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
282}
283
284Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, Register Op0) {
285 if (RetVT.SimpleTy != MVT::v4f16)
286 return Register();
287 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
288}
289
290Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, Register Op0) {
291 if (RetVT.SimpleTy != MVT::v8f16)
292 return Register();
293 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
294}
295
296Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, Register Op0) {
297 if (RetVT.SimpleTy != MVT::v4bf16)
298 return Register();
299 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
300}
301
302Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, Register Op0) {
303 if (RetVT.SimpleTy != MVT::v8bf16)
304 return Register();
305 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
306}
307
308Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, Register Op0) {
309 if (RetVT.SimpleTy != MVT::v2f32)
310 return Register();
311 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
312}
313
314Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, Register Op0) {
315 if (RetVT.SimpleTy != MVT::v4f32)
316 return Register();
317 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
318}
319
320Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, Register Op0) {
321 if (RetVT.SimpleTy != MVT::v1f64)
322 return Register();
323 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
324}
325
326Register fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, Register Op0) {
327 if (RetVT.SimpleTy != MVT::v2f64)
328 return Register();
329 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
330}
331
332Register fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, Register Op0) {
333 switch (VT.SimpleTy) {
334 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
335 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
336 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
337 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
338 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
339 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
340 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
341 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
342 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
343 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
344 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
345 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
346 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
347 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
348 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
349 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
350 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
351 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
352 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
353 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
354 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
355 default: return Register();
356 }
357}
358
359// FastEmit functions for AArch64ISD::DUP.
360
361Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Register Op0) {
362 if ((Subtarget->isNeonAvailable())) {
363 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
364 }
365 return Register();
366}
367
368Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Register Op0) {
369 if ((Subtarget->isNeonAvailable())) {
370 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
371 }
372 return Register();
373}
374
375Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Register Op0) {
376 if ((Subtarget->isNeonAvailable())) {
377 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
378 }
379 return Register();
380}
381
382Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Register Op0) {
383 if ((Subtarget->isNeonAvailable())) {
384 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
385 }
386 return Register();
387}
388
389Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Register Op0) {
390 if ((Subtarget->isNeonAvailable())) {
391 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
392 }
393 return Register();
394}
395
396Register fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Register Op0) {
397 if ((Subtarget->isNeonAvailable())) {
398 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
399 }
400 return Register();
401}
402
403Register fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, Register Op0) {
404switch (RetVT.SimpleTy) {
405 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
406 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
407 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
408 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
409 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
410 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
411 default: return Register();
412}
413}
414
415Register fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, Register Op0) {
416 if (RetVT.SimpleTy != MVT::v2i64)
417 return Register();
418 if ((Subtarget->isNeonAvailable())) {
419 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
420 }
421 return Register();
422}
423
424Register fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, Register Op0) {
425 switch (VT.SimpleTy) {
426 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
427 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
428 default: return Register();
429 }
430}
431
432// FastEmit functions for AArch64ISD::FCVTXN.
433
434Register fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, Register Op0) {
435 if (RetVT.SimpleTy != MVT::f32)
436 return Register();
437 if ((Subtarget->isNeonAvailable())) {
438 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
439 }
440 return Register();
441}
442
443Register fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, Register Op0) {
444 if (RetVT.SimpleTy != MVT::v2f32)
445 return Register();
446 if ((Subtarget->isNeonAvailable())) {
447 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
448 }
449 return Register();
450}
451
452Register fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, Register Op0) {
453 switch (VT.SimpleTy) {
454 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
455 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
456 default: return Register();
457 }
458}
459
460// FastEmit functions for AArch64ISD::FRECPE.
461
462Register fastEmit_AArch64ISD_FRECPE_MVT_f32_r(MVT RetVT, Register Op0) {
463 if (RetVT.SimpleTy != MVT::f32)
464 return Register();
465 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv1i32, RC: &AArch64::FPR32RegClass, Op0);
466}
467
468Register fastEmit_AArch64ISD_FRECPE_MVT_f64_r(MVT RetVT, Register Op0) {
469 if (RetVT.SimpleTy != MVT::f64)
470 return Register();
471 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv1i64, RC: &AArch64::FPR64RegClass, Op0);
472}
473
474Register fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, Register Op0) {
475 if (RetVT.SimpleTy != MVT::v2f32)
476 return Register();
477 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
478}
479
480Register fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, Register Op0) {
481 if (RetVT.SimpleTy != MVT::v4f32)
482 return Register();
483 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
484}
485
486Register fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(MVT RetVT, Register Op0) {
487 if (RetVT.SimpleTy != MVT::v1f64)
488 return Register();
489 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv1i64, RC: &AArch64::FPR64RegClass, Op0);
490}
491
492Register fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, Register Op0) {
493 if (RetVT.SimpleTy != MVT::v2f64)
494 return Register();
495 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
496}
497
498Register fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
499 if (RetVT.SimpleTy != MVT::nxv8f16)
500 return Register();
501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
502 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
503 }
504 return Register();
505}
506
507Register fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
508 if (RetVT.SimpleTy != MVT::nxv4f32)
509 return Register();
510 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
511 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
512 }
513 return Register();
514}
515
516Register fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
517 if (RetVT.SimpleTy != MVT::nxv2f64)
518 return Register();
519 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
520 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
521 }
522 return Register();
523}
524
525Register fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, Register Op0) {
526 switch (VT.SimpleTy) {
527 case MVT::f32: return fastEmit_AArch64ISD_FRECPE_MVT_f32_r(RetVT, Op0);
528 case MVT::f64: return fastEmit_AArch64ISD_FRECPE_MVT_f64_r(RetVT, Op0);
529 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
530 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
531 case MVT::v1f64: return fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(RetVT, Op0);
532 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
533 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
534 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
535 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
536 default: return Register();
537 }
538}
539
540// FastEmit functions for AArch64ISD::FRSQRTE.
541
542Register fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(MVT RetVT, Register Op0) {
543 if (RetVT.SimpleTy != MVT::f32)
544 return Register();
545 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv1i32, RC: &AArch64::FPR32RegClass, Op0);
546}
547
548Register fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(MVT RetVT, Register Op0) {
549 if (RetVT.SimpleTy != MVT::f64)
550 return Register();
551 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv1i64, RC: &AArch64::FPR64RegClass, Op0);
552}
553
554Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, Register Op0) {
555 if (RetVT.SimpleTy != MVT::v2f32)
556 return Register();
557 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
558}
559
560Register fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, Register Op0) {
561 if (RetVT.SimpleTy != MVT::v4f32)
562 return Register();
563 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
564}
565
566Register fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(MVT RetVT, Register Op0) {
567 if (RetVT.SimpleTy != MVT::v1f64)
568 return Register();
569 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv1i64, RC: &AArch64::FPR64RegClass, Op0);
570}
571
572Register fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, Register Op0) {
573 if (RetVT.SimpleTy != MVT::v2f64)
574 return Register();
575 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
576}
577
578Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
579 if (RetVT.SimpleTy != MVT::nxv8f16)
580 return Register();
581 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
582 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
583 }
584 return Register();
585}
586
587Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
588 if (RetVT.SimpleTy != MVT::nxv4f32)
589 return Register();
590 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
591 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
592 }
593 return Register();
594}
595
596Register fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
597 if (RetVT.SimpleTy != MVT::nxv2f64)
598 return Register();
599 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
600 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
601 }
602 return Register();
603}
604
605Register fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, Register Op0) {
606 switch (VT.SimpleTy) {
607 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(RetVT, Op0);
608 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(RetVT, Op0);
609 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
610 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
611 case MVT::v1f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(RetVT, Op0);
612 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
613 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
614 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
615 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
616 default: return Register();
617 }
618}
619
620// FastEmit functions for AArch64ISD::INIT_TPIDR2OBJ.
621
622Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_r(MVT RetVT, Register Op0) {
623 if (RetVT.SimpleTy != MVT::isVoid)
624 return Register();
625 return fastEmitInst_r(MachineInstOpcode: AArch64::InitTPIDR2Obj, RC: &AArch64::GPR64RegClass, Op0);
626}
627
628Register fastEmit_AArch64ISD_INIT_TPIDR2OBJ_r(MVT VT, MVT RetVT, Register Op0) {
629 switch (VT.SimpleTy) {
630 case MVT::i64: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_r(RetVT, Op0);
631 default: return Register();
632 }
633}
634
635// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
636
637Register fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, Register Op0) {
638 if (RetVT.SimpleTy != MVT::isVoid)
639 return Register();
640 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
641}
642
643Register fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, Register Op0) {
644 switch (VT.SimpleTy) {
645 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
646 default: return Register();
647 }
648}
649
650// FastEmit functions for AArch64ISD::REV16.
651
652Register fastEmit_AArch64ISD_REV16_MVT_i32_r(MVT RetVT, Register Op0) {
653 if (RetVT.SimpleTy != MVT::i32)
654 return Register();
655 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Wr, RC: &AArch64::GPR32RegClass, Op0);
656}
657
658Register fastEmit_AArch64ISD_REV16_MVT_i64_r(MVT RetVT, Register Op0) {
659 if (RetVT.SimpleTy != MVT::i64)
660 return Register();
661 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16Xr, RC: &AArch64::GPR64RegClass, Op0);
662}
663
664Register fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, Register Op0) {
665 if (RetVT.SimpleTy != MVT::v8i8)
666 return Register();
667 if ((Subtarget->isNeonAvailable())) {
668 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
669 }
670 return Register();
671}
672
673Register fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, Register Op0) {
674 if (RetVT.SimpleTy != MVT::v16i8)
675 return Register();
676 if ((Subtarget->isNeonAvailable())) {
677 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
678 }
679 return Register();
680}
681
682Register fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, Register Op0) {
683 switch (VT.SimpleTy) {
684 case MVT::i32: return fastEmit_AArch64ISD_REV16_MVT_i32_r(RetVT, Op0);
685 case MVT::i64: return fastEmit_AArch64ISD_REV16_MVT_i64_r(RetVT, Op0);
686 case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0);
687 case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0);
688 default: return Register();
689 }
690}
691
692// FastEmit functions for AArch64ISD::REV32.
693
694Register fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, Register Op0) {
695 if (RetVT.SimpleTy != MVT::v8i8)
696 return Register();
697 if ((Subtarget->isNeonAvailable())) {
698 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
699 }
700 return Register();
701}
702
703Register fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, Register Op0) {
704 if (RetVT.SimpleTy != MVT::v16i8)
705 return Register();
706 if ((Subtarget->isNeonAvailable())) {
707 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
708 }
709 return Register();
710}
711
712Register fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, Register Op0) {
713 if (RetVT.SimpleTy != MVT::v4i16)
714 return Register();
715 if ((Subtarget->isNeonAvailable())) {
716 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
717 }
718 return Register();
719}
720
721Register fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, Register Op0) {
722 if (RetVT.SimpleTy != MVT::v8i16)
723 return Register();
724 if ((Subtarget->isNeonAvailable())) {
725 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
726 }
727 return Register();
728}
729
730Register fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, Register Op0) {
731 if (RetVT.SimpleTy != MVT::v4f16)
732 return Register();
733 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
734}
735
736Register fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, Register Op0) {
737 if (RetVT.SimpleTy != MVT::v8f16)
738 return Register();
739 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
740}
741
742Register fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, Register Op0) {
743 if (RetVT.SimpleTy != MVT::v4bf16)
744 return Register();
745 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
746}
747
748Register fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, Register Op0) {
749 if (RetVT.SimpleTy != MVT::v8bf16)
750 return Register();
751 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
752}
753
754Register fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, Register Op0) {
755 switch (VT.SimpleTy) {
756 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
757 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
758 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
759 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
760 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
761 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
762 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
763 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
764 default: return Register();
765 }
766}
767
768// FastEmit functions for AArch64ISD::REV64.
769
770Register fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, Register Op0) {
771 if (RetVT.SimpleTy != MVT::v8i8)
772 return Register();
773 if ((Subtarget->isNeonAvailable())) {
774 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
775 }
776 return Register();
777}
778
779Register fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, Register Op0) {
780 if (RetVT.SimpleTy != MVT::v16i8)
781 return Register();
782 if ((Subtarget->isNeonAvailable())) {
783 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
784 }
785 return Register();
786}
787
788Register fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, Register Op0) {
789 if (RetVT.SimpleTy != MVT::v4i16)
790 return Register();
791 if ((Subtarget->isNeonAvailable())) {
792 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
793 }
794 return Register();
795}
796
797Register fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, Register Op0) {
798 if (RetVT.SimpleTy != MVT::v8i16)
799 return Register();
800 if ((Subtarget->isNeonAvailable())) {
801 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
802 }
803 return Register();
804}
805
806Register fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, Register Op0) {
807 if (RetVT.SimpleTy != MVT::v2i32)
808 return Register();
809 if ((Subtarget->isNeonAvailable())) {
810 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
811 }
812 return Register();
813}
814
815Register fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, Register Op0) {
816 if (RetVT.SimpleTy != MVT::v4i32)
817 return Register();
818 if ((Subtarget->isNeonAvailable())) {
819 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
820 }
821 return Register();
822}
823
824Register fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, Register Op0) {
825 if (RetVT.SimpleTy != MVT::v4f16)
826 return Register();
827 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
828}
829
830Register fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, Register Op0) {
831 if (RetVT.SimpleTy != MVT::v8f16)
832 return Register();
833 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
834}
835
836Register fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, Register Op0) {
837 if (RetVT.SimpleTy != MVT::v4bf16)
838 return Register();
839 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
840}
841
842Register fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, Register Op0) {
843 if (RetVT.SimpleTy != MVT::v8bf16)
844 return Register();
845 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
846}
847
848Register fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, Register Op0) {
849 if (RetVT.SimpleTy != MVT::v2f32)
850 return Register();
851 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
852}
853
854Register fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, Register Op0) {
855 if (RetVT.SimpleTy != MVT::v4f32)
856 return Register();
857 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
858}
859
860Register fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, Register Op0) {
861 switch (VT.SimpleTy) {
862 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
863 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
864 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
865 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
866 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
867 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
868 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
869 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
870 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
871 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
872 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
873 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
874 default: return Register();
875 }
876}
877
878// FastEmit functions for AArch64ISD::SADDLP.
879
880Register fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
881 if (RetVT.SimpleTy != MVT::v4i16)
882 return Register();
883 if ((Subtarget->isNeonAvailable())) {
884 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
885 }
886 return Register();
887}
888
889Register fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
890 if (RetVT.SimpleTy != MVT::v8i16)
891 return Register();
892 if ((Subtarget->isNeonAvailable())) {
893 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
894 }
895 return Register();
896}
897
898Register fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
899 if (RetVT.SimpleTy != MVT::v2i32)
900 return Register();
901 if ((Subtarget->isNeonAvailable())) {
902 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
903 }
904 return Register();
905}
906
907Register fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
908 if (RetVT.SimpleTy != MVT::v4i32)
909 return Register();
910 if ((Subtarget->isNeonAvailable())) {
911 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
912 }
913 return Register();
914}
915
916Register fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
917 if (RetVT.SimpleTy != MVT::v1i64)
918 return Register();
919 if ((Subtarget->isNeonAvailable())) {
920 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
921 }
922 return Register();
923}
924
925Register fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
926 if (RetVT.SimpleTy != MVT::v2i64)
927 return Register();
928 if ((Subtarget->isNeonAvailable())) {
929 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
930 }
931 return Register();
932}
933
934Register fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, Register Op0) {
935 switch (VT.SimpleTy) {
936 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
937 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
938 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
939 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
940 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
941 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
942 default: return Register();
943 }
944}
945
946// FastEmit functions for AArch64ISD::SITOF.
947
948Register fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, Register Op0) {
949 if (RetVT.SimpleTy != MVT::f16)
950 return Register();
951 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
952 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
953 }
954 return Register();
955}
956
957Register fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, Register Op0) {
958 if (RetVT.SimpleTy != MVT::f32)
959 return Register();
960 if ((Subtarget->hasNEON())) {
961 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
962 }
963 return Register();
964}
965
966Register fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, Register Op0) {
967 if (RetVT.SimpleTy != MVT::f64)
968 return Register();
969 if ((Subtarget->hasNEON())) {
970 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
971 }
972 return Register();
973}
974
975Register fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, Register Op0) {
976 switch (VT.SimpleTy) {
977 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
978 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
979 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
980 default: return Register();
981 }
982}
983
984// FastEmit functions for AArch64ISD::SUNPKHI.
985
986Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
987 if (RetVT.SimpleTy != MVT::nxv8i16)
988 return Register();
989 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
990 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
991 }
992 return Register();
993}
994
995Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
996 if (RetVT.SimpleTy != MVT::nxv4i32)
997 return Register();
998 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
999 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1000 }
1001 return Register();
1002}
1003
1004Register fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1005 if (RetVT.SimpleTy != MVT::nxv2i64)
1006 return Register();
1007 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1008 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1009 }
1010 return Register();
1011}
1012
1013Register fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1014 switch (VT.SimpleTy) {
1015 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1016 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1017 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1018 default: return Register();
1019 }
1020}
1021
1022// FastEmit functions for AArch64ISD::SUNPKLO.
1023
1024Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1025 if (RetVT.SimpleTy != MVT::nxv8i16)
1026 return Register();
1027 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1028 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1029 }
1030 return Register();
1031}
1032
1033Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1034 if (RetVT.SimpleTy != MVT::nxv4i32)
1035 return Register();
1036 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1037 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1038 }
1039 return Register();
1040}
1041
1042Register fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1043 if (RetVT.SimpleTy != MVT::nxv2i64)
1044 return Register();
1045 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1046 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1047 }
1048 return Register();
1049}
1050
1051Register fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1052 switch (VT.SimpleTy) {
1053 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1054 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1055 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1056 default: return Register();
1057 }
1058}
1059
1060// FastEmit functions for AArch64ISD::UADDLP.
1061
1062Register fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, Register Op0) {
1063 if (RetVT.SimpleTy != MVT::v4i16)
1064 return Register();
1065 if ((Subtarget->isNeonAvailable())) {
1066 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1067 }
1068 return Register();
1069}
1070
1071Register fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, Register Op0) {
1072 if (RetVT.SimpleTy != MVT::v8i16)
1073 return Register();
1074 if ((Subtarget->isNeonAvailable())) {
1075 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1076 }
1077 return Register();
1078}
1079
1080Register fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, Register Op0) {
1081 if (RetVT.SimpleTy != MVT::v2i32)
1082 return Register();
1083 if ((Subtarget->isNeonAvailable())) {
1084 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1085 }
1086 return Register();
1087}
1088
1089Register fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, Register Op0) {
1090 if (RetVT.SimpleTy != MVT::v4i32)
1091 return Register();
1092 if ((Subtarget->isNeonAvailable())) {
1093 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1094 }
1095 return Register();
1096}
1097
1098Register fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, Register Op0) {
1099 if (RetVT.SimpleTy != MVT::v1i64)
1100 return Register();
1101 if ((Subtarget->isNeonAvailable())) {
1102 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1103 }
1104 return Register();
1105}
1106
1107Register fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, Register Op0) {
1108 if (RetVT.SimpleTy != MVT::v2i64)
1109 return Register();
1110 if ((Subtarget->isNeonAvailable())) {
1111 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1112 }
1113 return Register();
1114}
1115
1116Register fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, Register Op0) {
1117 switch (VT.SimpleTy) {
1118 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1119 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1120 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1121 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1122 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1123 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1124 default: return Register();
1125 }
1126}
1127
1128// FastEmit functions for AArch64ISD::UITOF.
1129
1130Register fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, Register Op0) {
1131 if (RetVT.SimpleTy != MVT::f16)
1132 return Register();
1133 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
1134 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1135 }
1136 return Register();
1137}
1138
1139Register fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, Register Op0) {
1140 if (RetVT.SimpleTy != MVT::f32)
1141 return Register();
1142 if ((Subtarget->hasNEON())) {
1143 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1144 }
1145 return Register();
1146}
1147
1148Register fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, Register Op0) {
1149 if (RetVT.SimpleTy != MVT::f64)
1150 return Register();
1151 if ((Subtarget->hasNEON())) {
1152 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1153 }
1154 return Register();
1155}
1156
1157Register fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, Register Op0) {
1158 switch (VT.SimpleTy) {
1159 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1160 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1161 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1162 default: return Register();
1163 }
1164}
1165
1166// FastEmit functions for AArch64ISD::UUNPKHI.
1167
1168Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1169 if (RetVT.SimpleTy != MVT::nxv8i16)
1170 return Register();
1171 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1172 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1173 }
1174 return Register();
1175}
1176
1177Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1178 if (RetVT.SimpleTy != MVT::nxv4i32)
1179 return Register();
1180 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1181 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1182 }
1183 return Register();
1184}
1185
1186Register fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1187 if (RetVT.SimpleTy != MVT::nxv2i64)
1188 return Register();
1189 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1190 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1191 }
1192 return Register();
1193}
1194
1195Register fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, Register Op0) {
1196 switch (VT.SimpleTy) {
1197 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1198 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1199 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1200 default: return Register();
1201 }
1202}
1203
1204// FastEmit functions for AArch64ISD::UUNPKLO.
1205
1206Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
1207 if (RetVT.SimpleTy != MVT::nxv8i16)
1208 return Register();
1209 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1210 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1211 }
1212 return Register();
1213}
1214
1215Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
1216 if (RetVT.SimpleTy != MVT::nxv4i32)
1217 return Register();
1218 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1219 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1220 }
1221 return Register();
1222}
1223
1224Register fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
1225 if (RetVT.SimpleTy != MVT::nxv2i64)
1226 return Register();
1227 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
1228 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1229 }
1230 return Register();
1231}
1232
1233Register fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, Register Op0) {
1234 switch (VT.SimpleTy) {
1235 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1236 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1237 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1238 default: return Register();
1239 }
1240}
1241
1242// FastEmit functions for ISD::ABS.
1243
1244Register fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, Register Op0) {
1245 if (RetVT.SimpleTy != MVT::i32)
1246 return Register();
1247 if ((Subtarget->hasCSSC())) {
1248 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1249 }
1250 return Register();
1251}
1252
1253Register fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, Register Op0) {
1254 if (RetVT.SimpleTy != MVT::i64)
1255 return Register();
1256 if ((!Subtarget->hasCSSC())) {
1257 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1258 }
1259 if ((Subtarget->hasCSSC())) {
1260 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1261 }
1262 return Register();
1263}
1264
1265Register fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, Register Op0) {
1266 if (RetVT.SimpleTy != MVT::v8i8)
1267 return Register();
1268 if ((Subtarget->isNeonAvailable())) {
1269 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
1270 }
1271 return Register();
1272}
1273
1274Register fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, Register Op0) {
1275 if (RetVT.SimpleTy != MVT::v16i8)
1276 return Register();
1277 if ((Subtarget->isNeonAvailable())) {
1278 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
1279 }
1280 return Register();
1281}
1282
1283Register fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, Register Op0) {
1284 if (RetVT.SimpleTy != MVT::v4i16)
1285 return Register();
1286 if ((Subtarget->isNeonAvailable())) {
1287 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
1288 }
1289 return Register();
1290}
1291
1292Register fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, Register Op0) {
1293 if (RetVT.SimpleTy != MVT::v8i16)
1294 return Register();
1295 if ((Subtarget->isNeonAvailable())) {
1296 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
1297 }
1298 return Register();
1299}
1300
1301Register fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, Register Op0) {
1302 if (RetVT.SimpleTy != MVT::v2i32)
1303 return Register();
1304 if ((Subtarget->isNeonAvailable())) {
1305 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
1306 }
1307 return Register();
1308}
1309
1310Register fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, Register Op0) {
1311 if (RetVT.SimpleTy != MVT::v4i32)
1312 return Register();
1313 if ((Subtarget->isNeonAvailable())) {
1314 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
1315 }
1316 return Register();
1317}
1318
1319Register fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, Register Op0) {
1320 if (RetVT.SimpleTy != MVT::v1i64)
1321 return Register();
1322 if ((Subtarget->isNeonAvailable())) {
1323 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1324 }
1325 return Register();
1326}
1327
1328Register fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, Register Op0) {
1329 if (RetVT.SimpleTy != MVT::v2i64)
1330 return Register();
1331 if ((Subtarget->isNeonAvailable())) {
1332 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
1333 }
1334 return Register();
1335}
1336
1337Register fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, Register Op0) {
1338 switch (VT.SimpleTy) {
1339 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
1340 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
1341 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
1342 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
1343 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
1344 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
1345 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
1346 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
1347 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
1348 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
1349 default: return Register();
1350 }
1351}
1352
1353// FastEmit functions for ISD::BITCAST.
1354
1355Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Register Op0) {
1356 if ((!Subtarget->isLittleEndian())) {
1357 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1358 }
1359 return Register();
1360}
1361
1362Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Register Op0) {
1363 if ((!Subtarget->isLittleEndian())) {
1364 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1365 }
1366 return Register();
1367}
1368
1369Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Register Op0) {
1370 if ((!Subtarget->isLittleEndian())) {
1371 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1372 }
1373 return Register();
1374}
1375
1376Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Register Op0) {
1377 if ((!Subtarget->isLittleEndian())) {
1378 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1379 }
1380 return Register();
1381}
1382
1383Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Register Op0) {
1384 if ((!Subtarget->isLittleEndian())) {
1385 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1386 }
1387 return Register();
1388}
1389
1390Register fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Register Op0) {
1391 if ((!Subtarget->isLittleEndian())) {
1392 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1393 }
1394 return Register();
1395}
1396
1397Register fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, Register Op0) {
1398switch (RetVT.SimpleTy) {
1399 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
1400 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
1401 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
1402 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
1403 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
1404 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
1405 default: return Register();
1406}
1407}
1408
1409Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Register Op0) {
1410 if ((!Subtarget->isLittleEndian())) {
1411 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1412 }
1413 return Register();
1414}
1415
1416Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Register Op0) {
1417 if ((!Subtarget->isLittleEndian())) {
1418 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1419 }
1420 return Register();
1421}
1422
1423Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Register Op0) {
1424 if ((!Subtarget->isLittleEndian())) {
1425 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1426 }
1427 return Register();
1428}
1429
1430Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Register Op0) {
1431 if ((!Subtarget->isLittleEndian())) {
1432 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1433 }
1434 return Register();
1435}
1436
1437Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Register Op0) {
1438 if ((!Subtarget->isLittleEndian())) {
1439 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1440 }
1441 return Register();
1442}
1443
1444Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Register Op0) {
1445 if ((!Subtarget->isLittleEndian())) {
1446 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1447 }
1448 return Register();
1449}
1450
1451Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Register Op0) {
1452 if ((!Subtarget->isLittleEndian())) {
1453 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1454 }
1455 return Register();
1456}
1457
1458Register fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Register Op0) {
1459 if ((!Subtarget->isLittleEndian())) {
1460 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1461 }
1462 return Register();
1463}
1464
1465Register fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, Register Op0) {
1466switch (RetVT.SimpleTy) {
1467 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
1468 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
1469 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
1470 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
1471 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
1472 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
1473 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
1474 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
1475 default: return Register();
1476}
1477}
1478
1479Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Register Op0) {
1480 if ((!Subtarget->isLittleEndian())) {
1481 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1482 }
1483 return Register();
1484}
1485
1486Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Register Op0) {
1487 if ((!Subtarget->isLittleEndian())) {
1488 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1489 }
1490 return Register();
1491}
1492
1493Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Register Op0) {
1494 if ((!Subtarget->isLittleEndian())) {
1495 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1496 }
1497 return Register();
1498}
1499
1500Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Register Op0) {
1501 if ((!Subtarget->isLittleEndian())) {
1502 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1503 }
1504 return Register();
1505}
1506
1507Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Register Op0) {
1508 if ((!Subtarget->isLittleEndian())) {
1509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1510 }
1511 return Register();
1512}
1513
1514Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Register Op0) {
1515 if ((!Subtarget->isLittleEndian())) {
1516 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1517 }
1518 return Register();
1519}
1520
1521Register fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Register Op0) {
1522 if ((!Subtarget->isLittleEndian())) {
1523 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1524 }
1525 return Register();
1526}
1527
1528Register fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, Register Op0) {
1529switch (RetVT.SimpleTy) {
1530 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
1531 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
1532 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
1533 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
1534 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
1535 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
1536 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
1537 default: return Register();
1538}
1539}
1540
1541Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Register Op0) {
1542 if ((!Subtarget->isLittleEndian())) {
1543 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1544 }
1545 return Register();
1546}
1547
1548Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Register Op0) {
1549 if ((!Subtarget->isLittleEndian())) {
1550 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1551 }
1552 return Register();
1553}
1554
1555Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Register Op0) {
1556 if ((!Subtarget->isLittleEndian())) {
1557 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1558 }
1559 return Register();
1560}
1561
1562Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Register Op0) {
1563 if ((!Subtarget->isLittleEndian())) {
1564 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1565 }
1566 return Register();
1567}
1568
1569Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Register Op0) {
1570 if ((!Subtarget->isLittleEndian())) {
1571 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1572 }
1573 return Register();
1574}
1575
1576Register fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Register Op0) {
1577 if ((!Subtarget->isLittleEndian())) {
1578 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1579 }
1580 return Register();
1581}
1582
1583Register fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, Register Op0) {
1584switch (RetVT.SimpleTy) {
1585 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
1586 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
1587 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
1588 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
1589 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
1590 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
1591 default: return Register();
1592}
1593}
1594
1595Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Register Op0) {
1596 if ((!Subtarget->isLittleEndian())) {
1597 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1598 }
1599 return Register();
1600}
1601
1602Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Register Op0) {
1603 if ((!Subtarget->isLittleEndian())) {
1604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1605 }
1606 return Register();
1607}
1608
1609Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Register Op0) {
1610 if ((!Subtarget->isLittleEndian())) {
1611 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1612 }
1613 return Register();
1614}
1615
1616Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Register Op0) {
1617 if ((!Subtarget->isLittleEndian())) {
1618 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1619 }
1620 return Register();
1621}
1622
1623Register fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Register Op0) {
1624 if ((!Subtarget->isLittleEndian())) {
1625 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1626 }
1627 return Register();
1628}
1629
1630Register fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, Register Op0) {
1631switch (RetVT.SimpleTy) {
1632 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
1633 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
1634 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
1635 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
1636 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
1637 default: return Register();
1638}
1639}
1640
1641Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Register Op0) {
1642 if ((!Subtarget->isLittleEndian())) {
1643 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1644 }
1645 return Register();
1646}
1647
1648Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Register Op0) {
1649 if ((!Subtarget->isLittleEndian())) {
1650 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1651 }
1652 return Register();
1653}
1654
1655Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Register Op0) {
1656 if ((!Subtarget->isLittleEndian())) {
1657 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1658 }
1659 return Register();
1660}
1661
1662Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Register Op0) {
1663 if ((!Subtarget->isLittleEndian())) {
1664 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1665 }
1666 return Register();
1667}
1668
1669Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Register Op0) {
1670 if ((!Subtarget->isLittleEndian())) {
1671 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1672 }
1673 return Register();
1674}
1675
1676Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Register Op0) {
1677 if ((!Subtarget->isLittleEndian())) {
1678 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1679 }
1680 return Register();
1681}
1682
1683Register fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Register Op0) {
1684 if ((!Subtarget->isLittleEndian())) {
1685 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1686 }
1687 return Register();
1688}
1689
1690Register fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, Register Op0) {
1691switch (RetVT.SimpleTy) {
1692 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
1693 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
1694 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
1695 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
1696 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
1697 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
1698 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
1699 default: return Register();
1700}
1701}
1702
1703Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Register Op0) {
1704 if ((!Subtarget->isLittleEndian())) {
1705 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1706 }
1707 return Register();
1708}
1709
1710Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Register Op0) {
1711 if ((!Subtarget->isLittleEndian())) {
1712 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1713 }
1714 return Register();
1715}
1716
1717Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Register Op0) {
1718 if ((!Subtarget->isLittleEndian())) {
1719 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1720 }
1721 return Register();
1722}
1723
1724Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Register Op0) {
1725 if ((!Subtarget->isLittleEndian())) {
1726 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1727 }
1728 return Register();
1729}
1730
1731Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Register Op0) {
1732 if ((!Subtarget->isLittleEndian())) {
1733 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1734 }
1735 return Register();
1736}
1737
1738Register fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Register Op0) {
1739 if ((!Subtarget->isLittleEndian())) {
1740 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1741 }
1742 return Register();
1743}
1744
1745Register fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, Register Op0) {
1746switch (RetVT.SimpleTy) {
1747 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
1748 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
1749 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
1750 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
1751 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
1752 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
1753 default: return Register();
1754}
1755}
1756
1757Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Register Op0) {
1758 if ((!Subtarget->isLittleEndian())) {
1759 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1760 }
1761 return Register();
1762}
1763
1764Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Register Op0) {
1765 if ((!Subtarget->isLittleEndian())) {
1766 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1767 }
1768 return Register();
1769}
1770
1771Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Register Op0) {
1772 if ((!Subtarget->isLittleEndian())) {
1773 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1774 }
1775 return Register();
1776}
1777
1778Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Register Op0) {
1779 if ((!Subtarget->isLittleEndian())) {
1780 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1781 }
1782 return Register();
1783}
1784
1785Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Register Op0) {
1786 if ((!Subtarget->isLittleEndian())) {
1787 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1788 }
1789 return Register();
1790}
1791
1792Register fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Register Op0) {
1793 if ((!Subtarget->isLittleEndian())) {
1794 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1795 }
1796 return Register();
1797}
1798
1799Register fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, Register Op0) {
1800switch (RetVT.SimpleTy) {
1801 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
1802 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
1803 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
1804 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
1805 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
1806 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
1807 default: return Register();
1808}
1809}
1810
1811Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Register Op0) {
1812 if ((!Subtarget->isLittleEndian())) {
1813 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1814 }
1815 return Register();
1816}
1817
1818Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Register Op0) {
1819 if ((!Subtarget->isLittleEndian())) {
1820 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1821 }
1822 return Register();
1823}
1824
1825Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Register Op0) {
1826 if ((!Subtarget->isLittleEndian())) {
1827 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1828 }
1829 return Register();
1830}
1831
1832Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Register Op0) {
1833 if ((!Subtarget->isLittleEndian())) {
1834 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1835 }
1836 return Register();
1837}
1838
1839Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Register Op0) {
1840 if ((!Subtarget->isLittleEndian())) {
1841 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1842 }
1843 return Register();
1844}
1845
1846Register fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Register Op0) {
1847 if ((!Subtarget->isLittleEndian())) {
1848 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1849 }
1850 return Register();
1851}
1852
1853Register fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, Register Op0) {
1854switch (RetVT.SimpleTy) {
1855 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
1856 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
1857 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
1858 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
1859 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
1860 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
1861 default: return Register();
1862}
1863}
1864
1865Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Register Op0) {
1866 if ((!Subtarget->isLittleEndian())) {
1867 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1868 }
1869 return Register();
1870}
1871
1872Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Register Op0) {
1873 if ((!Subtarget->isLittleEndian())) {
1874 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1875 }
1876 return Register();
1877}
1878
1879Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Register Op0) {
1880 if ((!Subtarget->isLittleEndian())) {
1881 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1882 }
1883 return Register();
1884}
1885
1886Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Register Op0) {
1887 if ((!Subtarget->isLittleEndian())) {
1888 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1889 }
1890 return Register();
1891}
1892
1893Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Register Op0) {
1894 if ((!Subtarget->isLittleEndian())) {
1895 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1896 }
1897 return Register();
1898}
1899
1900Register fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Register Op0) {
1901 if ((!Subtarget->isLittleEndian())) {
1902 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1903 }
1904 return Register();
1905}
1906
1907Register fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, Register Op0) {
1908switch (RetVT.SimpleTy) {
1909 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
1910 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
1911 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
1912 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
1913 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
1914 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
1915 default: return Register();
1916}
1917}
1918
1919Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Register Op0) {
1920 if ((!Subtarget->isLittleEndian())) {
1921 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1922 }
1923 return Register();
1924}
1925
1926Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Register Op0) {
1927 if ((!Subtarget->isLittleEndian())) {
1928 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1929 }
1930 return Register();
1931}
1932
1933Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Register Op0) {
1934 if ((!Subtarget->isLittleEndian())) {
1935 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1936 }
1937 return Register();
1938}
1939
1940Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Register Op0) {
1941 if ((!Subtarget->isLittleEndian())) {
1942 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1943 }
1944 return Register();
1945}
1946
1947Register fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Register Op0) {
1948 if ((!Subtarget->isLittleEndian())) {
1949 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1950 }
1951 return Register();
1952}
1953
1954Register fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, Register Op0) {
1955switch (RetVT.SimpleTy) {
1956 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
1957 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
1958 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
1959 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
1960 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
1961 default: return Register();
1962}
1963}
1964
1965Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Register Op0) {
1966 if ((!Subtarget->isLittleEndian())) {
1967 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1968 }
1969 return Register();
1970}
1971
1972Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Register Op0) {
1973 if ((!Subtarget->isLittleEndian())) {
1974 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1975 }
1976 return Register();
1977}
1978
1979Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Register Op0) {
1980 if ((!Subtarget->isLittleEndian())) {
1981 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1982 }
1983 return Register();
1984}
1985
1986Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Register Op0) {
1987 if ((!Subtarget->isLittleEndian())) {
1988 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1989 }
1990 return Register();
1991}
1992
1993Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Register Op0) {
1994 if ((!Subtarget->isLittleEndian())) {
1995 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1996 }
1997 return Register();
1998}
1999
2000Register fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Register Op0) {
2001 if ((!Subtarget->isLittleEndian())) {
2002 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2003 }
2004 return Register();
2005}
2006
2007Register fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, Register Op0) {
2008switch (RetVT.SimpleTy) {
2009 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
2010 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
2011 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
2012 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
2013 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
2014 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
2015 default: return Register();
2016}
2017}
2018
2019Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Register Op0) {
2020 if ((!Subtarget->isLittleEndian())) {
2021 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2022 }
2023 return Register();
2024}
2025
2026Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Register Op0) {
2027 if ((!Subtarget->isLittleEndian())) {
2028 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2029 }
2030 return Register();
2031}
2032
2033Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Register Op0) {
2034 if ((!Subtarget->isLittleEndian())) {
2035 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2036 }
2037 return Register();
2038}
2039
2040Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Register Op0) {
2041 if ((!Subtarget->isLittleEndian())) {
2042 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2043 }
2044 return Register();
2045}
2046
2047Register fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Register Op0) {
2048 if ((!Subtarget->isLittleEndian())) {
2049 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2050 }
2051 return Register();
2052}
2053
2054Register fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, Register Op0) {
2055switch (RetVT.SimpleTy) {
2056 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
2057 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
2058 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
2059 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
2060 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
2061 default: return Register();
2062}
2063}
2064
2065Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Register Op0) {
2066 if ((!Subtarget->isLittleEndian())) {
2067 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2068 }
2069 return Register();
2070}
2071
2072Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Register Op0) {
2073 if ((!Subtarget->isLittleEndian())) {
2074 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2075 }
2076 return Register();
2077}
2078
2079Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Register Op0) {
2080 if ((!Subtarget->isLittleEndian())) {
2081 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2082 }
2083 return Register();
2084}
2085
2086Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Register Op0) {
2087 if ((!Subtarget->isLittleEndian())) {
2088 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2089 }
2090 return Register();
2091}
2092
2093Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Register Op0) {
2094 if ((!Subtarget->isLittleEndian())) {
2095 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2096 }
2097 return Register();
2098}
2099
2100Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Register Op0) {
2101 if ((!Subtarget->isLittleEndian())) {
2102 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2103 }
2104 return Register();
2105}
2106
2107Register fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Register Op0) {
2108 if ((!Subtarget->isLittleEndian())) {
2109 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2110 }
2111 return Register();
2112}
2113
2114Register fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, Register Op0) {
2115switch (RetVT.SimpleTy) {
2116 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2117 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2118 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2119 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2120 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2121 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2122 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2123 default: return Register();
2124}
2125}
2126
2127Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Register Op0) {
2128 if ((!Subtarget->isLittleEndian())) {
2129 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2130 }
2131 return Register();
2132}
2133
2134Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Register Op0) {
2135 if ((!Subtarget->isLittleEndian())) {
2136 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2137 }
2138 return Register();
2139}
2140
2141Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Register Op0) {
2142 if ((!Subtarget->isLittleEndian())) {
2143 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2144 }
2145 return Register();
2146}
2147
2148Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Register Op0) {
2149 if ((!Subtarget->isLittleEndian())) {
2150 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2151 }
2152 return Register();
2153}
2154
2155Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Register Op0) {
2156 if ((!Subtarget->isLittleEndian())) {
2157 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2158 }
2159 return Register();
2160}
2161
2162Register fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Register Op0) {
2163 if ((!Subtarget->isLittleEndian())) {
2164 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2165 }
2166 return Register();
2167}
2168
2169Register fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, Register Op0) {
2170switch (RetVT.SimpleTy) {
2171 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2172 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2173 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2174 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2175 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2176 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2177 default: return Register();
2178}
2179}
2180
2181Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Register Op0) {
2182 if ((!Subtarget->isLittleEndian())) {
2183 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2184 }
2185 return Register();
2186}
2187
2188Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Register Op0) {
2189 if ((!Subtarget->isLittleEndian())) {
2190 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2191 }
2192 return Register();
2193}
2194
2195Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Register Op0) {
2196 if ((!Subtarget->isLittleEndian())) {
2197 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2198 }
2199 return Register();
2200}
2201
2202Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Register Op0) {
2203 if ((!Subtarget->isLittleEndian())) {
2204 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2205 }
2206 return Register();
2207}
2208
2209Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Register Op0) {
2210 if ((!Subtarget->isLittleEndian())) {
2211 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2212 }
2213 return Register();
2214}
2215
2216Register fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Register Op0) {
2217 if ((!Subtarget->isLittleEndian())) {
2218 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2219 }
2220 return Register();
2221}
2222
2223Register fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, Register Op0) {
2224switch (RetVT.SimpleTy) {
2225 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2226 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2227 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2228 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2229 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2230 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2231 default: return Register();
2232}
2233}
2234
2235Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Register Op0) {
2236 if ((!Subtarget->isLittleEndian())) {
2237 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2238 }
2239 return Register();
2240}
2241
2242Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Register Op0) {
2243 if ((!Subtarget->isLittleEndian())) {
2244 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2245 }
2246 return Register();
2247}
2248
2249Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Register Op0) {
2250 if ((!Subtarget->isLittleEndian())) {
2251 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2252 }
2253 return Register();
2254}
2255
2256Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Register Op0) {
2257 if ((!Subtarget->isLittleEndian())) {
2258 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2259 }
2260 return Register();
2261}
2262
2263Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Register Op0) {
2264 if ((!Subtarget->isLittleEndian())) {
2265 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2266 }
2267 return Register();
2268}
2269
2270Register fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Register Op0) {
2271 if ((!Subtarget->isLittleEndian())) {
2272 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2273 }
2274 return Register();
2275}
2276
2277Register fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, Register Op0) {
2278switch (RetVT.SimpleTy) {
2279 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
2280 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
2281 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
2282 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
2283 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
2284 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
2285 default: return Register();
2286}
2287}
2288
2289Register fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, Register Op0) {
2290 switch (VT.SimpleTy) {
2291 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
2292 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
2293 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
2294 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
2295 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
2296 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
2297 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
2298 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
2299 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
2300 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
2301 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
2302 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
2303 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
2304 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
2305 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
2306 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
2307 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
2308 default: return Register();
2309 }
2310}
2311
2312// FastEmit functions for ISD::BITREVERSE.
2313
2314Register fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, Register Op0) {
2315 if (RetVT.SimpleTy != MVT::i32)
2316 return Register();
2317 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
2318}
2319
2320Register fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, Register Op0) {
2321 if (RetVT.SimpleTy != MVT::i64)
2322 return Register();
2323 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
2324}
2325
2326Register fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, Register Op0) {
2327 if (RetVT.SimpleTy != MVT::v8i8)
2328 return Register();
2329 if ((Subtarget->isNeonAvailable())) {
2330 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
2331 }
2332 return Register();
2333}
2334
2335Register fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, Register Op0) {
2336 if (RetVT.SimpleTy != MVT::v16i8)
2337 return Register();
2338 if ((Subtarget->isNeonAvailable())) {
2339 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
2340 }
2341 return Register();
2342}
2343
2344Register fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, Register Op0) {
2345 switch (VT.SimpleTy) {
2346 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
2347 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
2348 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
2349 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
2350 default: return Register();
2351 }
2352}
2353
2354// FastEmit functions for ISD::BRIND.
2355
2356Register fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, Register Op0) {
2357 if (RetVT.SimpleTy != MVT::isVoid)
2358 return Register();
2359 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
2360}
2361
2362Register fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, Register Op0) {
2363 switch (VT.SimpleTy) {
2364 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
2365 default: return Register();
2366 }
2367}
2368
2369// FastEmit functions for ISD::BSWAP.
2370
2371Register fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, Register Op0) {
2372 if (RetVT.SimpleTy != MVT::i32)
2373 return Register();
2374 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
2375}
2376
2377Register fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, Register Op0) {
2378 if (RetVT.SimpleTy != MVT::i64)
2379 return Register();
2380 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
2381}
2382
2383Register fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, Register Op0) {
2384 if (RetVT.SimpleTy != MVT::v4i16)
2385 return Register();
2386 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2387}
2388
2389Register fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, Register Op0) {
2390 if (RetVT.SimpleTy != MVT::v8i16)
2391 return Register();
2392 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2393}
2394
2395Register fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, Register Op0) {
2396 if (RetVT.SimpleTy != MVT::v2i32)
2397 return Register();
2398 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2399}
2400
2401Register fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, Register Op0) {
2402 if (RetVT.SimpleTy != MVT::v4i32)
2403 return Register();
2404 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2405}
2406
2407Register fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, Register Op0) {
2408 if (RetVT.SimpleTy != MVT::v2i64)
2409 return Register();
2410 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2411}
2412
2413Register fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, Register Op0) {
2414 switch (VT.SimpleTy) {
2415 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
2416 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
2417 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
2418 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
2419 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
2420 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
2421 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
2422 default: return Register();
2423 }
2424}
2425
2426// FastEmit functions for ISD::CTLZ.
2427
2428Register fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, Register Op0) {
2429 if (RetVT.SimpleTy != MVT::i32)
2430 return Register();
2431 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
2432}
2433
2434Register fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, Register Op0) {
2435 if (RetVT.SimpleTy != MVT::i64)
2436 return Register();
2437 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
2438}
2439
2440Register fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, Register Op0) {
2441 if (RetVT.SimpleTy != MVT::v8i8)
2442 return Register();
2443 if ((Subtarget->isNeonAvailable())) {
2444 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
2445 }
2446 return Register();
2447}
2448
2449Register fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, Register Op0) {
2450 if (RetVT.SimpleTy != MVT::v16i8)
2451 return Register();
2452 if ((Subtarget->isNeonAvailable())) {
2453 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
2454 }
2455 return Register();
2456}
2457
2458Register fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, Register Op0) {
2459 if (RetVT.SimpleTy != MVT::v4i16)
2460 return Register();
2461 if ((Subtarget->isNeonAvailable())) {
2462 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
2463 }
2464 return Register();
2465}
2466
2467Register fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, Register Op0) {
2468 if (RetVT.SimpleTy != MVT::v8i16)
2469 return Register();
2470 if ((Subtarget->isNeonAvailable())) {
2471 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
2472 }
2473 return Register();
2474}
2475
2476Register fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, Register Op0) {
2477 if (RetVT.SimpleTy != MVT::v2i32)
2478 return Register();
2479 if ((Subtarget->isNeonAvailable())) {
2480 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
2481 }
2482 return Register();
2483}
2484
2485Register fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, Register Op0) {
2486 if (RetVT.SimpleTy != MVT::v4i32)
2487 return Register();
2488 if ((Subtarget->isNeonAvailable())) {
2489 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
2490 }
2491 return Register();
2492}
2493
2494Register fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, Register Op0) {
2495 switch (VT.SimpleTy) {
2496 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
2497 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
2498 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
2499 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
2500 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
2501 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
2502 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
2503 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
2504 default: return Register();
2505 }
2506}
2507
2508// FastEmit functions for ISD::CTPOP.
2509
2510Register fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, Register Op0) {
2511 if (RetVT.SimpleTy != MVT::i32)
2512 return Register();
2513 if ((Subtarget->hasCSSC())) {
2514 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
2515 }
2516 return Register();
2517}
2518
2519Register fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, Register Op0) {
2520 if (RetVT.SimpleTy != MVT::i64)
2521 return Register();
2522 if ((Subtarget->hasCSSC())) {
2523 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
2524 }
2525 return Register();
2526}
2527
2528Register fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, Register Op0) {
2529 if (RetVT.SimpleTy != MVT::v8i8)
2530 return Register();
2531 if ((Subtarget->isNeonAvailable())) {
2532 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
2533 }
2534 return Register();
2535}
2536
2537Register fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, Register Op0) {
2538 if (RetVT.SimpleTy != MVT::v16i8)
2539 return Register();
2540 if ((Subtarget->isNeonAvailable())) {
2541 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
2542 }
2543 return Register();
2544}
2545
2546Register fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, Register Op0) {
2547 switch (VT.SimpleTy) {
2548 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
2549 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
2550 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
2551 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
2552 default: return Register();
2553 }
2554}
2555
2556// FastEmit functions for ISD::CTTZ.
2557
2558Register fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, Register Op0) {
2559 if (RetVT.SimpleTy != MVT::i32)
2560 return Register();
2561 if ((Subtarget->hasCSSC())) {
2562 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
2563 }
2564 return Register();
2565}
2566
2567Register fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, Register Op0) {
2568 if (RetVT.SimpleTy != MVT::i64)
2569 return Register();
2570 if ((Subtarget->hasCSSC())) {
2571 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
2572 }
2573 return Register();
2574}
2575
2576Register fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, Register Op0) {
2577 switch (VT.SimpleTy) {
2578 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
2579 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
2580 default: return Register();
2581 }
2582}
2583
2584// FastEmit functions for ISD::FABS.
2585
2586Register fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, Register Op0) {
2587 if (RetVT.SimpleTy != MVT::f16)
2588 return Register();
2589 if ((Subtarget->hasFullFP16())) {
2590 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
2591 }
2592 return Register();
2593}
2594
2595Register fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, Register Op0) {
2596 if (RetVT.SimpleTy != MVT::f32)
2597 return Register();
2598 if ((Subtarget->hasFPARMv8())) {
2599 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
2600 }
2601 return Register();
2602}
2603
2604Register fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, Register Op0) {
2605 if (RetVT.SimpleTy != MVT::f64)
2606 return Register();
2607 if ((Subtarget->hasFPARMv8())) {
2608 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
2609 }
2610 return Register();
2611}
2612
2613Register fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, Register Op0) {
2614 if (RetVT.SimpleTy != MVT::v4f16)
2615 return Register();
2616 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2617 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
2618 }
2619 return Register();
2620}
2621
2622Register fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, Register Op0) {
2623 if (RetVT.SimpleTy != MVT::v8f16)
2624 return Register();
2625 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2626 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
2627 }
2628 return Register();
2629}
2630
2631Register fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, Register Op0) {
2632 if (RetVT.SimpleTy != MVT::v2f32)
2633 return Register();
2634 if ((Subtarget->isNeonAvailable())) {
2635 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
2636 }
2637 return Register();
2638}
2639
2640Register fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, Register Op0) {
2641 if (RetVT.SimpleTy != MVT::v4f32)
2642 return Register();
2643 if ((Subtarget->isNeonAvailable())) {
2644 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
2645 }
2646 return Register();
2647}
2648
2649Register fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, Register Op0) {
2650 if (RetVT.SimpleTy != MVT::v2f64)
2651 return Register();
2652 if ((Subtarget->isNeonAvailable())) {
2653 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
2654 }
2655 return Register();
2656}
2657
2658Register fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, Register Op0) {
2659 switch (VT.SimpleTy) {
2660 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
2661 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
2662 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
2663 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
2664 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
2665 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
2666 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
2667 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
2668 default: return Register();
2669 }
2670}
2671
2672// FastEmit functions for ISD::FCEIL.
2673
2674Register fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
2675 if (RetVT.SimpleTy != MVT::f16)
2676 return Register();
2677 if ((Subtarget->hasFullFP16())) {
2678 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
2679 }
2680 return Register();
2681}
2682
2683Register fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
2684 if (RetVT.SimpleTy != MVT::f32)
2685 return Register();
2686 if ((Subtarget->hasFPARMv8())) {
2687 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
2688 }
2689 return Register();
2690}
2691
2692Register fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
2693 if (RetVT.SimpleTy != MVT::f64)
2694 return Register();
2695 if ((Subtarget->hasFPARMv8())) {
2696 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
2697 }
2698 return Register();
2699}
2700
2701Register fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
2702 if (RetVT.SimpleTy != MVT::v4f16)
2703 return Register();
2704 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2705 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
2706 }
2707 return Register();
2708}
2709
2710Register fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
2711 if (RetVT.SimpleTy != MVT::v8f16)
2712 return Register();
2713 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2714 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
2715 }
2716 return Register();
2717}
2718
2719Register fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
2720 if (RetVT.SimpleTy != MVT::v2f32)
2721 return Register();
2722 if ((Subtarget->isNeonAvailable())) {
2723 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
2724 }
2725 return Register();
2726}
2727
2728Register fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
2729 if (RetVT.SimpleTy != MVT::v4f32)
2730 return Register();
2731 if ((Subtarget->isNeonAvailable())) {
2732 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
2733 }
2734 return Register();
2735}
2736
2737Register fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
2738 if (RetVT.SimpleTy != MVT::v2f64)
2739 return Register();
2740 if ((Subtarget->isNeonAvailable())) {
2741 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
2742 }
2743 return Register();
2744}
2745
2746Register fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
2747 switch (VT.SimpleTy) {
2748 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
2749 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
2750 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
2751 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
2752 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
2753 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
2754 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
2755 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
2756 default: return Register();
2757 }
2758}
2759
2760// FastEmit functions for ISD::FFLOOR.
2761
2762Register fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
2763 if (RetVT.SimpleTy != MVT::f16)
2764 return Register();
2765 if ((Subtarget->hasFullFP16())) {
2766 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
2767 }
2768 return Register();
2769}
2770
2771Register fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
2772 if (RetVT.SimpleTy != MVT::f32)
2773 return Register();
2774 if ((Subtarget->hasFPARMv8())) {
2775 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
2776 }
2777 return Register();
2778}
2779
2780Register fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
2781 if (RetVT.SimpleTy != MVT::f64)
2782 return Register();
2783 if ((Subtarget->hasFPARMv8())) {
2784 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
2785 }
2786 return Register();
2787}
2788
2789Register fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
2790 if (RetVT.SimpleTy != MVT::v4f16)
2791 return Register();
2792 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2793 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
2794 }
2795 return Register();
2796}
2797
2798Register fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
2799 if (RetVT.SimpleTy != MVT::v8f16)
2800 return Register();
2801 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2802 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
2803 }
2804 return Register();
2805}
2806
2807Register fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
2808 if (RetVT.SimpleTy != MVT::v2f32)
2809 return Register();
2810 if ((Subtarget->isNeonAvailable())) {
2811 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
2812 }
2813 return Register();
2814}
2815
2816Register fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
2817 if (RetVT.SimpleTy != MVT::v4f32)
2818 return Register();
2819 if ((Subtarget->isNeonAvailable())) {
2820 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
2821 }
2822 return Register();
2823}
2824
2825Register fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
2826 if (RetVT.SimpleTy != MVT::v2f64)
2827 return Register();
2828 if ((Subtarget->isNeonAvailable())) {
2829 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
2830 }
2831 return Register();
2832}
2833
2834Register fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
2835 switch (VT.SimpleTy) {
2836 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
2837 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
2838 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
2839 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
2840 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
2841 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
2842 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
2843 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
2844 default: return Register();
2845 }
2846}
2847
2848// FastEmit functions for ISD::FNEARBYINT.
2849
2850Register fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
2851 if (RetVT.SimpleTy != MVT::f16)
2852 return Register();
2853 if ((Subtarget->hasFullFP16())) {
2854 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
2855 }
2856 return Register();
2857}
2858
2859Register fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
2860 if (RetVT.SimpleTy != MVT::f32)
2861 return Register();
2862 if ((Subtarget->hasFPARMv8())) {
2863 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
2864 }
2865 return Register();
2866}
2867
2868Register fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
2869 if (RetVT.SimpleTy != MVT::f64)
2870 return Register();
2871 if ((Subtarget->hasFPARMv8())) {
2872 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
2873 }
2874 return Register();
2875}
2876
2877Register fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
2878 if (RetVT.SimpleTy != MVT::v4f16)
2879 return Register();
2880 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2881 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
2882 }
2883 return Register();
2884}
2885
2886Register fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
2887 if (RetVT.SimpleTy != MVT::v8f16)
2888 return Register();
2889 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2890 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
2891 }
2892 return Register();
2893}
2894
2895Register fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
2896 if (RetVT.SimpleTy != MVT::v2f32)
2897 return Register();
2898 if ((Subtarget->isNeonAvailable())) {
2899 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
2900 }
2901 return Register();
2902}
2903
2904Register fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
2905 if (RetVT.SimpleTy != MVT::v4f32)
2906 return Register();
2907 if ((Subtarget->isNeonAvailable())) {
2908 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
2909 }
2910 return Register();
2911}
2912
2913Register fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
2914 if (RetVT.SimpleTy != MVT::v2f64)
2915 return Register();
2916 if ((Subtarget->isNeonAvailable())) {
2917 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
2918 }
2919 return Register();
2920}
2921
2922Register fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
2923 switch (VT.SimpleTy) {
2924 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
2925 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
2926 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
2927 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
2928 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
2929 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
2930 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
2931 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
2932 default: return Register();
2933 }
2934}
2935
2936// FastEmit functions for ISD::FNEG.
2937
2938Register fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, Register Op0) {
2939 if (RetVT.SimpleTy != MVT::f16)
2940 return Register();
2941 if ((Subtarget->hasFullFP16())) {
2942 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
2943 }
2944 return Register();
2945}
2946
2947Register fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, Register Op0) {
2948 if (RetVT.SimpleTy != MVT::f32)
2949 return Register();
2950 if ((Subtarget->hasFPARMv8())) {
2951 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
2952 }
2953 return Register();
2954}
2955
2956Register fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, Register Op0) {
2957 if (RetVT.SimpleTy != MVT::f64)
2958 return Register();
2959 if ((Subtarget->hasFPARMv8())) {
2960 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
2961 }
2962 return Register();
2963}
2964
2965Register fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, Register Op0) {
2966 if (RetVT.SimpleTy != MVT::v4f16)
2967 return Register();
2968 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2969 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
2970 }
2971 return Register();
2972}
2973
2974Register fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, Register Op0) {
2975 if (RetVT.SimpleTy != MVT::v8f16)
2976 return Register();
2977 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
2978 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
2979 }
2980 return Register();
2981}
2982
2983Register fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, Register Op0) {
2984 if (RetVT.SimpleTy != MVT::v2f32)
2985 return Register();
2986 if ((Subtarget->isNeonAvailable())) {
2987 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
2988 }
2989 return Register();
2990}
2991
2992Register fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, Register Op0) {
2993 if (RetVT.SimpleTy != MVT::v4f32)
2994 return Register();
2995 if ((Subtarget->isNeonAvailable())) {
2996 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
2997 }
2998 return Register();
2999}
3000
3001Register fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, Register Op0) {
3002 if (RetVT.SimpleTy != MVT::v2f64)
3003 return Register();
3004 if ((Subtarget->isNeonAvailable())) {
3005 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3006 }
3007 return Register();
3008}
3009
3010Register fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, Register Op0) {
3011 switch (VT.SimpleTy) {
3012 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3013 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3014 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3015 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3016 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3017 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3018 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3019 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3020 default: return Register();
3021 }
3022}
3023
3024// FastEmit functions for ISD::FP_EXTEND.
3025
3026Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
3027 if ((Subtarget->hasFPARMv8())) {
3028 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3029 }
3030 return Register();
3031}
3032
3033Register fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
3034 if ((Subtarget->hasFPARMv8())) {
3035 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3036 }
3037 return Register();
3038}
3039
3040Register fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
3041switch (RetVT.SimpleTy) {
3042 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3043 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3044 default: return Register();
3045}
3046}
3047
3048Register fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
3049 if (RetVT.SimpleTy != MVT::f64)
3050 return Register();
3051 if ((Subtarget->hasFPARMv8())) {
3052 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3053 }
3054 return Register();
3055}
3056
3057Register fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3058 if (RetVT.SimpleTy != MVT::v4f32)
3059 return Register();
3060 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3061}
3062
3063Register fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
3064 if (RetVT.SimpleTy != MVT::v4f32)
3065 return Register();
3066 if ((Subtarget->isNeonAvailable())) {
3067 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3068 }
3069 return Register();
3070}
3071
3072Register fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3073 if (RetVT.SimpleTy != MVT::v2f64)
3074 return Register();
3075 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3076}
3077
3078Register fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
3079 switch (VT.SimpleTy) {
3080 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3081 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3082 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3083 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3084 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3085 default: return Register();
3086 }
3087}
3088
3089// FastEmit functions for ISD::FP_ROUND.
3090
3091Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
3092 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3093 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3094 }
3095 return Register();
3096}
3097
3098Register fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
3099 if ((Subtarget->hasFPARMv8())) {
3100 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3101 }
3102 return Register();
3103}
3104
3105Register fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3106switch (RetVT.SimpleTy) {
3107 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3108 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3109 default: return Register();
3110}
3111}
3112
3113Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
3114 if ((Subtarget->hasFPARMv8())) {
3115 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3116 }
3117 return Register();
3118}
3119
3120Register fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
3121 if ((Subtarget->hasFPARMv8())) {
3122 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3123 }
3124 return Register();
3125}
3126
3127Register fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3128switch (RetVT.SimpleTy) {
3129 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3130 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3131 default: return Register();
3132}
3133}
3134
3135Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
3136 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3137}
3138
3139Register fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
3140 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
3141 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
3142 }
3143 return Register();
3144}
3145
3146Register fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3147switch (RetVT.SimpleTy) {
3148 case MVT::v4f16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
3149 case MVT::v4bf16: return fastEmit_ISD_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
3150 default: return Register();
3151}
3152}
3153
3154Register fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3155 if (RetVT.SimpleTy != MVT::v2f32)
3156 return Register();
3157 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3158}
3159
3160Register fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
3161 switch (VT.SimpleTy) {
3162 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3163 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3164 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3165 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3166 default: return Register();
3167 }
3168}
3169
3170// FastEmit functions for ISD::FP_TO_SINT.
3171
3172Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
3173 if ((Subtarget->hasFullFP16())) {
3174 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3175 }
3176 return Register();
3177}
3178
3179Register fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
3180 if ((Subtarget->hasFullFP16())) {
3181 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3182 }
3183 return Register();
3184}
3185
3186Register fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
3187switch (RetVT.SimpleTy) {
3188 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3189 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3190 default: return Register();
3191}
3192}
3193
3194Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
3195 if ((Subtarget->hasFPARMv8())) {
3196 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3197 }
3198 return Register();
3199}
3200
3201Register fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
3202 if ((Subtarget->hasFPARMv8())) {
3203 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3204 }
3205 return Register();
3206}
3207
3208Register fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
3209switch (RetVT.SimpleTy) {
3210 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3211 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3212 default: return Register();
3213}
3214}
3215
3216Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
3217 if ((Subtarget->hasFPARMv8())) {
3218 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3219 }
3220 return Register();
3221}
3222
3223Register fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
3224 if ((Subtarget->hasFPARMv8())) {
3225 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3226 }
3227 return Register();
3228}
3229
3230Register fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
3231switch (RetVT.SimpleTy) {
3232 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3233 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3234 default: return Register();
3235}
3236}
3237
3238Register fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3239 if (RetVT.SimpleTy != MVT::v4i16)
3240 return Register();
3241 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3242 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3243 }
3244 return Register();
3245}
3246
3247Register fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3248 if (RetVT.SimpleTy != MVT::v8i16)
3249 return Register();
3250 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3251 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3252 }
3253 return Register();
3254}
3255
3256Register fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3257 if (RetVT.SimpleTy != MVT::v2i32)
3258 return Register();
3259 if ((Subtarget->isNeonAvailable())) {
3260 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3261 }
3262 return Register();
3263}
3264
3265Register fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3266 if (RetVT.SimpleTy != MVT::v4i32)
3267 return Register();
3268 if ((Subtarget->isNeonAvailable())) {
3269 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3270 }
3271 return Register();
3272}
3273
3274Register fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3275 if (RetVT.SimpleTy != MVT::v2i64)
3276 return Register();
3277 if ((Subtarget->isNeonAvailable())) {
3278 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3279 }
3280 return Register();
3281}
3282
3283Register fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
3284 switch (VT.SimpleTy) {
3285 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
3286 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
3287 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
3288 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
3289 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
3290 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
3291 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
3292 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
3293 default: return Register();
3294 }
3295}
3296
3297// FastEmit functions for ISD::FP_TO_SINT_SAT.
3298
3299Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3300 if ((Subtarget->hasFullFP16())) {
3301 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3302 }
3303 return Register();
3304}
3305
3306Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3307 if ((Subtarget->hasFullFP16())) {
3308 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3309 }
3310 return Register();
3311}
3312
3313Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3314switch (RetVT.SimpleTy) {
3315 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i32_r(Op0);
3316 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_MVT_i64_r(Op0);
3317 default: return Register();
3318}
3319}
3320
3321Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3322 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3323}
3324
3325Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3326 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3327}
3328
3329Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3330switch (RetVT.SimpleTy) {
3331 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i32_r(Op0);
3332 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_MVT_i64_r(Op0);
3333 default: return Register();
3334}
3335}
3336
3337Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3338 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3339}
3340
3341Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3342 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3343}
3344
3345Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3346switch (RetVT.SimpleTy) {
3347 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i32_r(Op0);
3348 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_MVT_i64_r(Op0);
3349 default: return Register();
3350}
3351}
3352
3353Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3354 if (RetVT.SimpleTy != MVT::v4i16)
3355 return Register();
3356 if ((Subtarget->hasFullFP16())) {
3357 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3358 }
3359 return Register();
3360}
3361
3362Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3363 if (RetVT.SimpleTy != MVT::v8i16)
3364 return Register();
3365 if ((Subtarget->hasFullFP16())) {
3366 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3367 }
3368 return Register();
3369}
3370
3371Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3372 if (RetVT.SimpleTy != MVT::v2i32)
3373 return Register();
3374 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3375}
3376
3377Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3378 if (RetVT.SimpleTy != MVT::v4i32)
3379 return Register();
3380 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3381}
3382
3383Register fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3384 if (RetVT.SimpleTy != MVT::v2i64)
3385 return Register();
3386 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3387}
3388
3389Register fastEmit_ISD_FP_TO_SINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3390 switch (VT.SimpleTy) {
3391 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f16_r(RetVT, Op0);
3392 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f32_r(RetVT, Op0);
3393 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_f64_r(RetVT, Op0);
3394 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f16_r(RetVT, Op0);
3395 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v8f16_r(RetVT, Op0);
3396 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f32_r(RetVT, Op0);
3397 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v4f32_r(RetVT, Op0);
3398 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_SAT_MVT_v2f64_r(RetVT, Op0);
3399 default: return Register();
3400 }
3401}
3402
3403// FastEmit functions for ISD::FP_TO_UINT.
3404
3405Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
3406 if ((Subtarget->hasFullFP16())) {
3407 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3408 }
3409 return Register();
3410}
3411
3412Register fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
3413 if ((Subtarget->hasFullFP16())) {
3414 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3415 }
3416 return Register();
3417}
3418
3419Register fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
3420switch (RetVT.SimpleTy) {
3421 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
3422 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
3423 default: return Register();
3424}
3425}
3426
3427Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
3428 if ((Subtarget->hasFPARMv8())) {
3429 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3430 }
3431 return Register();
3432}
3433
3434Register fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
3435 if ((Subtarget->hasFPARMv8())) {
3436 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3437 }
3438 return Register();
3439}
3440
3441Register fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
3442switch (RetVT.SimpleTy) {
3443 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
3444 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
3445 default: return Register();
3446}
3447}
3448
3449Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
3450 if ((Subtarget->hasFPARMv8())) {
3451 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3452 }
3453 return Register();
3454}
3455
3456Register fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
3457 if ((Subtarget->hasFPARMv8())) {
3458 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3459 }
3460 return Register();
3461}
3462
3463Register fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
3464switch (RetVT.SimpleTy) {
3465 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
3466 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
3467 default: return Register();
3468}
3469}
3470
3471Register fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3472 if (RetVT.SimpleTy != MVT::v4i16)
3473 return Register();
3474 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3475 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3476 }
3477 return Register();
3478}
3479
3480Register fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3481 if (RetVT.SimpleTy != MVT::v8i16)
3482 return Register();
3483 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3484 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3485 }
3486 return Register();
3487}
3488
3489Register fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3490 if (RetVT.SimpleTy != MVT::v2i32)
3491 return Register();
3492 if ((Subtarget->isNeonAvailable())) {
3493 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3494 }
3495 return Register();
3496}
3497
3498Register fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3499 if (RetVT.SimpleTy != MVT::v4i32)
3500 return Register();
3501 if ((Subtarget->isNeonAvailable())) {
3502 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3503 }
3504 return Register();
3505}
3506
3507Register fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3508 if (RetVT.SimpleTy != MVT::v2i64)
3509 return Register();
3510 if ((Subtarget->isNeonAvailable())) {
3511 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3512 }
3513 return Register();
3514}
3515
3516Register fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
3517 switch (VT.SimpleTy) {
3518 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
3519 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
3520 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
3521 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
3522 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
3523 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
3524 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
3525 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
3526 default: return Register();
3527 }
3528}
3529
3530// FastEmit functions for ISD::FP_TO_UINT_SAT.
3531
3532Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Register Op0) {
3533 if ((Subtarget->hasFullFP16())) {
3534 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
3535 }
3536 return Register();
3537}
3538
3539Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Register Op0) {
3540 if ((Subtarget->hasFullFP16())) {
3541 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
3542 }
3543 return Register();
3544}
3545
3546Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(MVT RetVT, Register Op0) {
3547switch (RetVT.SimpleTy) {
3548 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i32_r(Op0);
3549 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_MVT_i64_r(Op0);
3550 default: return Register();
3551}
3552}
3553
3554Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Register Op0) {
3555 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
3556}
3557
3558Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Register Op0) {
3559 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
3560}
3561
3562Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(MVT RetVT, Register Op0) {
3563switch (RetVT.SimpleTy) {
3564 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i32_r(Op0);
3565 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_MVT_i64_r(Op0);
3566 default: return Register();
3567}
3568}
3569
3570Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Register Op0) {
3571 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
3572}
3573
3574Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Register Op0) {
3575 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
3576}
3577
3578Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(MVT RetVT, Register Op0) {
3579switch (RetVT.SimpleTy) {
3580 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i32_r(Op0);
3581 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_MVT_i64_r(Op0);
3582 default: return Register();
3583}
3584}
3585
3586Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3587 if (RetVT.SimpleTy != MVT::v4i16)
3588 return Register();
3589 if ((Subtarget->hasFullFP16())) {
3590 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
3591 }
3592 return Register();
3593}
3594
3595Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3596 if (RetVT.SimpleTy != MVT::v8i16)
3597 return Register();
3598 if ((Subtarget->hasFullFP16())) {
3599 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
3600 }
3601 return Register();
3602}
3603
3604Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3605 if (RetVT.SimpleTy != MVT::v2i32)
3606 return Register();
3607 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
3608}
3609
3610Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3611 if (RetVT.SimpleTy != MVT::v4i32)
3612 return Register();
3613 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
3614}
3615
3616Register fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3617 if (RetVT.SimpleTy != MVT::v2i64)
3618 return Register();
3619 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
3620}
3621
3622Register fastEmit_ISD_FP_TO_UINT_SAT_r(MVT VT, MVT RetVT, Register Op0) {
3623 switch (VT.SimpleTy) {
3624 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f16_r(RetVT, Op0);
3625 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f32_r(RetVT, Op0);
3626 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_f64_r(RetVT, Op0);
3627 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f16_r(RetVT, Op0);
3628 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v8f16_r(RetVT, Op0);
3629 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f32_r(RetVT, Op0);
3630 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v4f32_r(RetVT, Op0);
3631 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_SAT_MVT_v2f64_r(RetVT, Op0);
3632 default: return Register();
3633 }
3634}
3635
3636// FastEmit functions for ISD::FRINT.
3637
3638Register fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
3639 if (RetVT.SimpleTy != MVT::f16)
3640 return Register();
3641 if ((Subtarget->hasFullFP16())) {
3642 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
3643 }
3644 return Register();
3645}
3646
3647Register fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
3648 if (RetVT.SimpleTy != MVT::f32)
3649 return Register();
3650 if ((Subtarget->hasFPARMv8())) {
3651 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
3652 }
3653 return Register();
3654}
3655
3656Register fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
3657 if (RetVT.SimpleTy != MVT::f64)
3658 return Register();
3659 if ((Subtarget->hasFPARMv8())) {
3660 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
3661 }
3662 return Register();
3663}
3664
3665Register fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3666 if (RetVT.SimpleTy != MVT::v4f16)
3667 return Register();
3668 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3669 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
3670 }
3671 return Register();
3672}
3673
3674Register fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3675 if (RetVT.SimpleTy != MVT::v8f16)
3676 return Register();
3677 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3678 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
3679 }
3680 return Register();
3681}
3682
3683Register fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3684 if (RetVT.SimpleTy != MVT::v2f32)
3685 return Register();
3686 if ((Subtarget->isNeonAvailable())) {
3687 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
3688 }
3689 return Register();
3690}
3691
3692Register fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3693 if (RetVT.SimpleTy != MVT::v4f32)
3694 return Register();
3695 if ((Subtarget->isNeonAvailable())) {
3696 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
3697 }
3698 return Register();
3699}
3700
3701Register fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3702 if (RetVT.SimpleTy != MVT::v2f64)
3703 return Register();
3704 if ((Subtarget->isNeonAvailable())) {
3705 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
3706 }
3707 return Register();
3708}
3709
3710Register fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
3711 switch (VT.SimpleTy) {
3712 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
3713 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
3714 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
3715 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
3716 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
3717 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
3718 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
3719 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
3720 default: return Register();
3721 }
3722}
3723
3724// FastEmit functions for ISD::FROUND.
3725
3726Register fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
3727 if (RetVT.SimpleTy != MVT::f16)
3728 return Register();
3729 if ((Subtarget->hasFullFP16())) {
3730 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
3731 }
3732 return Register();
3733}
3734
3735Register fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
3736 if (RetVT.SimpleTy != MVT::f32)
3737 return Register();
3738 if ((Subtarget->hasFPARMv8())) {
3739 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
3740 }
3741 return Register();
3742}
3743
3744Register fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
3745 if (RetVT.SimpleTy != MVT::f64)
3746 return Register();
3747 if ((Subtarget->hasFPARMv8())) {
3748 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
3749 }
3750 return Register();
3751}
3752
3753Register fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
3754 if (RetVT.SimpleTy != MVT::v4f16)
3755 return Register();
3756 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3757 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
3758 }
3759 return Register();
3760}
3761
3762Register fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
3763 if (RetVT.SimpleTy != MVT::v8f16)
3764 return Register();
3765 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3766 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
3767 }
3768 return Register();
3769}
3770
3771Register fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
3772 if (RetVT.SimpleTy != MVT::v2f32)
3773 return Register();
3774 if ((Subtarget->isNeonAvailable())) {
3775 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
3776 }
3777 return Register();
3778}
3779
3780Register fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
3781 if (RetVT.SimpleTy != MVT::v4f32)
3782 return Register();
3783 if ((Subtarget->isNeonAvailable())) {
3784 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
3785 }
3786 return Register();
3787}
3788
3789Register fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
3790 if (RetVT.SimpleTy != MVT::v2f64)
3791 return Register();
3792 if ((Subtarget->isNeonAvailable())) {
3793 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
3794 }
3795 return Register();
3796}
3797
3798Register fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
3799 switch (VT.SimpleTy) {
3800 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
3801 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
3802 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
3803 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
3804 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
3805 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
3806 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
3807 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
3808 default: return Register();
3809 }
3810}
3811
3812// FastEmit functions for ISD::FROUNDEVEN.
3813
3814Register fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
3815 if (RetVT.SimpleTy != MVT::f16)
3816 return Register();
3817 if ((Subtarget->hasFullFP16())) {
3818 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
3819 }
3820 return Register();
3821}
3822
3823Register fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
3824 if (RetVT.SimpleTy != MVT::f32)
3825 return Register();
3826 if ((Subtarget->hasFPARMv8())) {
3827 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
3828 }
3829 return Register();
3830}
3831
3832Register fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
3833 if (RetVT.SimpleTy != MVT::f64)
3834 return Register();
3835 if ((Subtarget->hasFPARMv8())) {
3836 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
3837 }
3838 return Register();
3839}
3840
3841Register fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
3842 if (RetVT.SimpleTy != MVT::v4f16)
3843 return Register();
3844 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3845 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
3846 }
3847 return Register();
3848}
3849
3850Register fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
3851 if (RetVT.SimpleTy != MVT::v8f16)
3852 return Register();
3853 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3854 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
3855 }
3856 return Register();
3857}
3858
3859Register fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
3860 if (RetVT.SimpleTy != MVT::v2f32)
3861 return Register();
3862 if ((Subtarget->isNeonAvailable())) {
3863 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
3864 }
3865 return Register();
3866}
3867
3868Register fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
3869 if (RetVT.SimpleTy != MVT::v4f32)
3870 return Register();
3871 if ((Subtarget->isNeonAvailable())) {
3872 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
3873 }
3874 return Register();
3875}
3876
3877Register fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
3878 if (RetVT.SimpleTy != MVT::v2f64)
3879 return Register();
3880 if ((Subtarget->isNeonAvailable())) {
3881 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
3882 }
3883 return Register();
3884}
3885
3886Register fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
3887 switch (VT.SimpleTy) {
3888 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
3889 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
3890 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
3891 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
3892 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
3893 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
3894 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
3895 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
3896 default: return Register();
3897 }
3898}
3899
3900// FastEmit functions for ISD::FSQRT.
3901
3902Register fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
3903 if (RetVT.SimpleTy != MVT::f16)
3904 return Register();
3905 if ((Subtarget->hasFullFP16())) {
3906 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
3907 }
3908 return Register();
3909}
3910
3911Register fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
3912 if (RetVT.SimpleTy != MVT::f32)
3913 return Register();
3914 if ((Subtarget->hasFPARMv8())) {
3915 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
3916 }
3917 return Register();
3918}
3919
3920Register fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
3921 if (RetVT.SimpleTy != MVT::f64)
3922 return Register();
3923 if ((Subtarget->hasFPARMv8())) {
3924 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
3925 }
3926 return Register();
3927}
3928
3929Register fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
3930 if (RetVT.SimpleTy != MVT::v4f16)
3931 return Register();
3932 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3933 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
3934 }
3935 return Register();
3936}
3937
3938Register fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
3939 if (RetVT.SimpleTy != MVT::v8f16)
3940 return Register();
3941 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3942 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
3943 }
3944 return Register();
3945}
3946
3947Register fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
3948 if (RetVT.SimpleTy != MVT::v2f32)
3949 return Register();
3950 if ((Subtarget->isNeonAvailable())) {
3951 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
3952 }
3953 return Register();
3954}
3955
3956Register fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
3957 if (RetVT.SimpleTy != MVT::v4f32)
3958 return Register();
3959 if ((Subtarget->isNeonAvailable())) {
3960 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
3961 }
3962 return Register();
3963}
3964
3965Register fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
3966 if (RetVT.SimpleTy != MVT::v2f64)
3967 return Register();
3968 if ((Subtarget->isNeonAvailable())) {
3969 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
3970 }
3971 return Register();
3972}
3973
3974Register fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
3975 switch (VT.SimpleTy) {
3976 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
3977 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
3978 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
3979 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
3980 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
3981 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
3982 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
3983 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
3984 default: return Register();
3985 }
3986}
3987
3988// FastEmit functions for ISD::FTRUNC.
3989
3990Register fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
3991 if (RetVT.SimpleTy != MVT::f16)
3992 return Register();
3993 if ((Subtarget->hasFullFP16())) {
3994 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
3995 }
3996 return Register();
3997}
3998
3999Register fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
4000 if (RetVT.SimpleTy != MVT::f32)
4001 return Register();
4002 if ((Subtarget->hasFPARMv8())) {
4003 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
4004 }
4005 return Register();
4006}
4007
4008Register fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
4009 if (RetVT.SimpleTy != MVT::f64)
4010 return Register();
4011 if ((Subtarget->hasFPARMv8())) {
4012 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
4013 }
4014 return Register();
4015}
4016
4017Register fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
4018 if (RetVT.SimpleTy != MVT::v4f16)
4019 return Register();
4020 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4021 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
4022 }
4023 return Register();
4024}
4025
4026Register fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
4027 if (RetVT.SimpleTy != MVT::v8f16)
4028 return Register();
4029 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4030 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
4031 }
4032 return Register();
4033}
4034
4035Register fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
4036 if (RetVT.SimpleTy != MVT::v2f32)
4037 return Register();
4038 if ((Subtarget->isNeonAvailable())) {
4039 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
4040 }
4041 return Register();
4042}
4043
4044Register fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
4045 if (RetVT.SimpleTy != MVT::v4f32)
4046 return Register();
4047 if ((Subtarget->isNeonAvailable())) {
4048 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
4049 }
4050 return Register();
4051}
4052
4053Register fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
4054 if (RetVT.SimpleTy != MVT::v2f64)
4055 return Register();
4056 if ((Subtarget->isNeonAvailable())) {
4057 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
4058 }
4059 return Register();
4060}
4061
4062Register fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
4063 switch (VT.SimpleTy) {
4064 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
4065 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
4066 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
4067 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
4068 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
4069 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
4070 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
4071 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
4072 default: return Register();
4073 }
4074}
4075
4076// FastEmit functions for ISD::LLROUND.
4077
4078Register fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4079 if (RetVT.SimpleTy != MVT::i64)
4080 return Register();
4081 if ((Subtarget->hasFullFP16())) {
4082 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4083 }
4084 return Register();
4085}
4086
4087Register fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4088 if (RetVT.SimpleTy != MVT::i64)
4089 return Register();
4090 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4091}
4092
4093Register fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4094 if (RetVT.SimpleTy != MVT::i64)
4095 return Register();
4096 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4097}
4098
4099Register fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
4100 switch (VT.SimpleTy) {
4101 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
4102 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
4103 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
4104 default: return Register();
4105 }
4106}
4107
4108// FastEmit functions for ISD::LROUND.
4109
4110Register fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
4111 if ((Subtarget->hasFullFP16())) {
4112 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
4113 }
4114 return Register();
4115}
4116
4117Register fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
4118 if ((Subtarget->hasFullFP16())) {
4119 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4120 }
4121 return Register();
4122}
4123
4124Register fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
4125switch (RetVT.SimpleTy) {
4126 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4127 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4128 default: return Register();
4129}
4130}
4131
4132Register fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
4133 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4134}
4135
4136Register fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
4137 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4138}
4139
4140Register fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4141switch (RetVT.SimpleTy) {
4142 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4143 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4144 default: return Register();
4145}
4146}
4147
4148Register fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
4149 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4150}
4151
4152Register fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
4153 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4154}
4155
4156Register fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4157switch (RetVT.SimpleTy) {
4158 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4159 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4160 default: return Register();
4161}
4162}
4163
4164Register fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
4165 switch (VT.SimpleTy) {
4166 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4167 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4168 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4169 default: return Register();
4170 }
4171}
4172
4173// FastEmit functions for ISD::SINT_TO_FP.
4174
4175Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
4176 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4177 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
4178 }
4179 if ((Subtarget->hasFullFP16())) {
4180 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4181 }
4182 return Register();
4183}
4184
4185Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
4186 if ((Subtarget->hasFPARMv8())) {
4187 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4188 }
4189 return Register();
4190}
4191
4192Register fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
4193 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4194 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
4195 }
4196 if ((Subtarget->hasFPARMv8())) {
4197 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4198 }
4199 return Register();
4200}
4201
4202Register fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
4203switch (RetVT.SimpleTy) {
4204 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4205 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4206 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4207 default: return Register();
4208}
4209}
4210
4211Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
4212 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4213 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
4214 }
4215 if ((Subtarget->hasFullFP16())) {
4216 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4217 }
4218 return Register();
4219}
4220
4221Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
4222 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
4223 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
4224 }
4225 if ((Subtarget->hasFPARMv8())) {
4226 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4227 }
4228 return Register();
4229}
4230
4231Register fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
4232 if ((Subtarget->hasFPARMv8())) {
4233 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4234 }
4235 return Register();
4236}
4237
4238Register fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
4239switch (RetVT.SimpleTy) {
4240 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4241 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4242 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4243 default: return Register();
4244}
4245}
4246
4247Register fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
4248 if (RetVT.SimpleTy != MVT::v4f16)
4249 return Register();
4250 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4251 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4252 }
4253 return Register();
4254}
4255
4256Register fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
4257 if (RetVT.SimpleTy != MVT::v8f16)
4258 return Register();
4259 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4260 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4261 }
4262 return Register();
4263}
4264
4265Register fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
4266 if (RetVT.SimpleTy != MVT::v2f32)
4267 return Register();
4268 if ((Subtarget->isNeonAvailable())) {
4269 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4270 }
4271 return Register();
4272}
4273
4274Register fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
4275 if (RetVT.SimpleTy != MVT::v4f32)
4276 return Register();
4277 if ((Subtarget->isNeonAvailable())) {
4278 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4279 }
4280 return Register();
4281}
4282
4283Register fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
4284 if (RetVT.SimpleTy != MVT::v2f64)
4285 return Register();
4286 if ((Subtarget->isNeonAvailable())) {
4287 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4288 }
4289 return Register();
4290}
4291
4292Register fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
4293 switch (VT.SimpleTy) {
4294 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4295 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4296 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4297 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4298 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4299 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4300 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4301 default: return Register();
4302 }
4303}
4304
4305// FastEmit functions for ISD::SPLAT_VECTOR.
4306
4307Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Register Op0) {
4308 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4309 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4310 }
4311 return Register();
4312}
4313
4314Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Register Op0) {
4315 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4316 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4317 }
4318 return Register();
4319}
4320
4321Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Register Op0) {
4322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4323 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4324 }
4325 return Register();
4326}
4327
4328Register fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, Register Op0) {
4329switch (RetVT.SimpleTy) {
4330 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4331 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4332 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4333 default: return Register();
4334}
4335}
4336
4337Register fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, Register Op0) {
4338 if (RetVT.SimpleTy != MVT::nxv2i64)
4339 return Register();
4340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
4341 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4342 }
4343 return Register();
4344}
4345
4346Register fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, Register Op0) {
4347 switch (VT.SimpleTy) {
4348 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4349 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4350 default: return Register();
4351 }
4352}
4353
4354// FastEmit functions for ISD::STRICT_FCEIL.
4355
4356Register fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, Register Op0) {
4357 if (RetVT.SimpleTy != MVT::f16)
4358 return Register();
4359 if ((Subtarget->hasFullFP16())) {
4360 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4361 }
4362 return Register();
4363}
4364
4365Register fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, Register Op0) {
4366 if (RetVT.SimpleTy != MVT::f32)
4367 return Register();
4368 if ((Subtarget->hasFPARMv8())) {
4369 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4370 }
4371 return Register();
4372}
4373
4374Register fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, Register Op0) {
4375 if (RetVT.SimpleTy != MVT::f64)
4376 return Register();
4377 if ((Subtarget->hasFPARMv8())) {
4378 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4379 }
4380 return Register();
4381}
4382
4383Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, Register Op0) {
4384 if (RetVT.SimpleTy != MVT::v4f16)
4385 return Register();
4386 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4387 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4388 }
4389 return Register();
4390}
4391
4392Register fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, Register Op0) {
4393 if (RetVT.SimpleTy != MVT::v8f16)
4394 return Register();
4395 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4396 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4397 }
4398 return Register();
4399}
4400
4401Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, Register Op0) {
4402 if (RetVT.SimpleTy != MVT::v2f32)
4403 return Register();
4404 if ((Subtarget->isNeonAvailable())) {
4405 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4406 }
4407 return Register();
4408}
4409
4410Register fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, Register Op0) {
4411 if (RetVT.SimpleTy != MVT::v4f32)
4412 return Register();
4413 if ((Subtarget->isNeonAvailable())) {
4414 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4415 }
4416 return Register();
4417}
4418
4419Register fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, Register Op0) {
4420 if (RetVT.SimpleTy != MVT::v2f64)
4421 return Register();
4422 if ((Subtarget->isNeonAvailable())) {
4423 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4424 }
4425 return Register();
4426}
4427
4428Register fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, Register Op0) {
4429 switch (VT.SimpleTy) {
4430 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4431 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4432 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4433 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4434 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4435 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4436 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4437 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4438 default: return Register();
4439 }
4440}
4441
4442// FastEmit functions for ISD::STRICT_FFLOOR.
4443
4444Register fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, Register Op0) {
4445 if (RetVT.SimpleTy != MVT::f16)
4446 return Register();
4447 if ((Subtarget->hasFullFP16())) {
4448 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4449 }
4450 return Register();
4451}
4452
4453Register fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, Register Op0) {
4454 if (RetVT.SimpleTy != MVT::f32)
4455 return Register();
4456 if ((Subtarget->hasFPARMv8())) {
4457 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4458 }
4459 return Register();
4460}
4461
4462Register fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, Register Op0) {
4463 if (RetVT.SimpleTy != MVT::f64)
4464 return Register();
4465 if ((Subtarget->hasFPARMv8())) {
4466 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4467 }
4468 return Register();
4469}
4470
4471Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, Register Op0) {
4472 if (RetVT.SimpleTy != MVT::v4f16)
4473 return Register();
4474 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4475 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4476 }
4477 return Register();
4478}
4479
4480Register fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, Register Op0) {
4481 if (RetVT.SimpleTy != MVT::v8f16)
4482 return Register();
4483 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4484 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4485 }
4486 return Register();
4487}
4488
4489Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, Register Op0) {
4490 if (RetVT.SimpleTy != MVT::v2f32)
4491 return Register();
4492 if ((Subtarget->isNeonAvailable())) {
4493 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4494 }
4495 return Register();
4496}
4497
4498Register fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, Register Op0) {
4499 if (RetVT.SimpleTy != MVT::v4f32)
4500 return Register();
4501 if ((Subtarget->isNeonAvailable())) {
4502 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4503 }
4504 return Register();
4505}
4506
4507Register fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, Register Op0) {
4508 if (RetVT.SimpleTy != MVT::v2f64)
4509 return Register();
4510 if ((Subtarget->isNeonAvailable())) {
4511 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
4512 }
4513 return Register();
4514}
4515
4516Register fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, Register Op0) {
4517 switch (VT.SimpleTy) {
4518 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
4519 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
4520 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
4521 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
4522 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
4523 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
4524 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
4525 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
4526 default: return Register();
4527 }
4528}
4529
4530// FastEmit functions for ISD::STRICT_FNEARBYINT.
4531
4532Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, Register Op0) {
4533 if (RetVT.SimpleTy != MVT::f16)
4534 return Register();
4535 if ((Subtarget->hasFullFP16())) {
4536 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
4537 }
4538 return Register();
4539}
4540
4541Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, Register Op0) {
4542 if (RetVT.SimpleTy != MVT::f32)
4543 return Register();
4544 if ((Subtarget->hasFPARMv8())) {
4545 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
4546 }
4547 return Register();
4548}
4549
4550Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, Register Op0) {
4551 if (RetVT.SimpleTy != MVT::f64)
4552 return Register();
4553 if ((Subtarget->hasFPARMv8())) {
4554 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
4555 }
4556 return Register();
4557}
4558
4559Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4560 if (RetVT.SimpleTy != MVT::v4f16)
4561 return Register();
4562 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4563 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
4564 }
4565 return Register();
4566}
4567
4568Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4569 if (RetVT.SimpleTy != MVT::v8f16)
4570 return Register();
4571 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4572 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
4573 }
4574 return Register();
4575}
4576
4577Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4578 if (RetVT.SimpleTy != MVT::v2f32)
4579 return Register();
4580 if ((Subtarget->isNeonAvailable())) {
4581 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
4582 }
4583 return Register();
4584}
4585
4586Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4587 if (RetVT.SimpleTy != MVT::v4f32)
4588 return Register();
4589 if ((Subtarget->isNeonAvailable())) {
4590 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
4591 }
4592 return Register();
4593}
4594
4595Register fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4596 if (RetVT.SimpleTy != MVT::v2f64)
4597 return Register();
4598 if ((Subtarget->isNeonAvailable())) {
4599 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
4600 }
4601 return Register();
4602}
4603
4604Register fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, Register Op0) {
4605 switch (VT.SimpleTy) {
4606 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
4607 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
4608 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
4609 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
4610 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
4611 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
4612 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
4613 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
4614 default: return Register();
4615 }
4616}
4617
4618// FastEmit functions for ISD::STRICT_FP_EXTEND.
4619
4620Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Register Op0) {
4621 if ((Subtarget->hasFPARMv8())) {
4622 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
4623 }
4624 return Register();
4625}
4626
4627Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Register Op0) {
4628 if ((Subtarget->hasFPARMv8())) {
4629 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
4630 }
4631 return Register();
4632}
4633
4634Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, Register Op0) {
4635switch (RetVT.SimpleTy) {
4636 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
4637 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
4638 default: return Register();
4639}
4640}
4641
4642Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, Register Op0) {
4643 if (RetVT.SimpleTy != MVT::f64)
4644 return Register();
4645 if ((Subtarget->hasFPARMv8())) {
4646 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
4647 }
4648 return Register();
4649}
4650
4651Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, Register Op0) {
4652 if (RetVT.SimpleTy != MVT::v4f32)
4653 return Register();
4654 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4655}
4656
4657Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, Register Op0) {
4658 if (RetVT.SimpleTy != MVT::v4f32)
4659 return Register();
4660 if ((Subtarget->isNeonAvailable())) {
4661 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
4662 }
4663 return Register();
4664}
4665
4666Register fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, Register Op0) {
4667 if (RetVT.SimpleTy != MVT::v2f64)
4668 return Register();
4669 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
4670}
4671
4672Register fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, Register Op0) {
4673 switch (VT.SimpleTy) {
4674 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
4675 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
4676 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
4677 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
4678 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
4679 default: return Register();
4680 }
4681}
4682
4683// FastEmit functions for ISD::STRICT_FP_ROUND.
4684
4685Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Register Op0) {
4686 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
4687 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
4688 }
4689 return Register();
4690}
4691
4692Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Register Op0) {
4693 if ((Subtarget->hasFPARMv8())) {
4694 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
4695 }
4696 return Register();
4697}
4698
4699Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, Register Op0) {
4700switch (RetVT.SimpleTy) {
4701 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
4702 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
4703 default: return Register();
4704}
4705}
4706
4707Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Register Op0) {
4708 if ((Subtarget->hasFPARMv8())) {
4709 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
4710 }
4711 return Register();
4712}
4713
4714Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Register Op0) {
4715 if ((Subtarget->hasFPARMv8())) {
4716 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
4717 }
4718 return Register();
4719}
4720
4721Register fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, Register Op0) {
4722switch (RetVT.SimpleTy) {
4723 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
4724 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
4725 default: return Register();
4726}
4727}
4728
4729Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Register Op0) {
4730 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
4731}
4732
4733Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Register Op0) {
4734 if ((Subtarget->hasBF16()) && (Subtarget->isNeonAvailable())) {
4735 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVTN, RC: &AArch64::FPR64RegClass, Op0);
4736 }
4737 return Register();
4738}
4739
4740Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
4741switch (RetVT.SimpleTy) {
4742 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4f16_r(Op0);
4743 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_MVT_v4bf16_r(Op0);
4744 default: return Register();
4745}
4746}
4747
4748Register fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
4749 if (RetVT.SimpleTy != MVT::v2f32)
4750 return Register();
4751 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
4752}
4753
4754Register fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, Register Op0) {
4755 switch (VT.SimpleTy) {
4756 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
4757 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
4758 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
4759 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
4760 default: return Register();
4761 }
4762}
4763
4764// FastEmit functions for ISD::STRICT_FP_TO_SINT.
4765
4766Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Register Op0) {
4767 if ((Subtarget->hasFullFP16())) {
4768 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
4769 }
4770 return Register();
4771}
4772
4773Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Register Op0) {
4774 if ((Subtarget->hasFullFP16())) {
4775 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
4776 }
4777 return Register();
4778}
4779
4780Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, Register Op0) {
4781switch (RetVT.SimpleTy) {
4782 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
4783 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
4784 default: return Register();
4785}
4786}
4787
4788Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Register Op0) {
4789 if ((Subtarget->hasFPARMv8())) {
4790 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
4791 }
4792 return Register();
4793}
4794
4795Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Register Op0) {
4796 if ((Subtarget->hasFPARMv8())) {
4797 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
4798 }
4799 return Register();
4800}
4801
4802Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, Register Op0) {
4803switch (RetVT.SimpleTy) {
4804 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
4805 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
4806 default: return Register();
4807}
4808}
4809
4810Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Register Op0) {
4811 if ((Subtarget->hasFPARMv8())) {
4812 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
4813 }
4814 return Register();
4815}
4816
4817Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Register Op0) {
4818 if ((Subtarget->hasFPARMv8())) {
4819 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
4820 }
4821 return Register();
4822}
4823
4824Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, Register Op0) {
4825switch (RetVT.SimpleTy) {
4826 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
4827 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
4828 default: return Register();
4829}
4830}
4831
4832Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4833 if (RetVT.SimpleTy != MVT::v4i16)
4834 return Register();
4835 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4836 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
4837 }
4838 return Register();
4839}
4840
4841Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4842 if (RetVT.SimpleTy != MVT::v8i16)
4843 return Register();
4844 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4845 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
4846 }
4847 return Register();
4848}
4849
4850Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4851 if (RetVT.SimpleTy != MVT::v2i32)
4852 return Register();
4853 if ((Subtarget->isNeonAvailable())) {
4854 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
4855 }
4856 return Register();
4857}
4858
4859Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4860 if (RetVT.SimpleTy != MVT::v4i32)
4861 return Register();
4862 if ((Subtarget->isNeonAvailable())) {
4863 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
4864 }
4865 return Register();
4866}
4867
4868Register fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4869 if (RetVT.SimpleTy != MVT::v2i64)
4870 return Register();
4871 if ((Subtarget->isNeonAvailable())) {
4872 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
4873 }
4874 return Register();
4875}
4876
4877Register fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, Register Op0) {
4878 switch (VT.SimpleTy) {
4879 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4880 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4881 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4882 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4883 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4884 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4885 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4886 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4887 default: return Register();
4888 }
4889}
4890
4891// FastEmit functions for ISD::STRICT_FP_TO_UINT.
4892
4893Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Register Op0) {
4894 if ((Subtarget->hasFullFP16())) {
4895 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4896 }
4897 return Register();
4898}
4899
4900Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Register Op0) {
4901 if ((Subtarget->hasFullFP16())) {
4902 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4903 }
4904 return Register();
4905}
4906
4907Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, Register Op0) {
4908switch (RetVT.SimpleTy) {
4909 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4910 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4911 default: return Register();
4912}
4913}
4914
4915Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Register Op0) {
4916 if ((Subtarget->hasFPARMv8())) {
4917 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4918 }
4919 return Register();
4920}
4921
4922Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Register Op0) {
4923 if ((Subtarget->hasFPARMv8())) {
4924 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4925 }
4926 return Register();
4927}
4928
4929Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, Register Op0) {
4930switch (RetVT.SimpleTy) {
4931 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4932 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4933 default: return Register();
4934}
4935}
4936
4937Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Register Op0) {
4938 if ((Subtarget->hasFPARMv8())) {
4939 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4940 }
4941 return Register();
4942}
4943
4944Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Register Op0) {
4945 if ((Subtarget->hasFPARMv8())) {
4946 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4947 }
4948 return Register();
4949}
4950
4951Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, Register Op0) {
4952switch (RetVT.SimpleTy) {
4953 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4954 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4955 default: return Register();
4956}
4957}
4958
4959Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
4960 if (RetVT.SimpleTy != MVT::v4i16)
4961 return Register();
4962 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4963 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4964 }
4965 return Register();
4966}
4967
4968Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
4969 if (RetVT.SimpleTy != MVT::v8i16)
4970 return Register();
4971 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4972 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4973 }
4974 return Register();
4975}
4976
4977Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
4978 if (RetVT.SimpleTy != MVT::v2i32)
4979 return Register();
4980 if ((Subtarget->isNeonAvailable())) {
4981 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4982 }
4983 return Register();
4984}
4985
4986Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
4987 if (RetVT.SimpleTy != MVT::v4i32)
4988 return Register();
4989 if ((Subtarget->isNeonAvailable())) {
4990 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4991 }
4992 return Register();
4993}
4994
4995Register fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
4996 if (RetVT.SimpleTy != MVT::v2i64)
4997 return Register();
4998 if ((Subtarget->isNeonAvailable())) {
4999 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
5000 }
5001 return Register();
5002}
5003
5004Register fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, Register Op0) {
5005 switch (VT.SimpleTy) {
5006 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
5007 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
5008 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
5009 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
5010 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
5011 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
5012 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
5013 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
5014 default: return Register();
5015 }
5016}
5017
5018// FastEmit functions for ISD::STRICT_FRINT.
5019
5020Register fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, Register Op0) {
5021 if (RetVT.SimpleTy != MVT::f16)
5022 return Register();
5023 if ((Subtarget->hasFullFP16())) {
5024 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
5025 }
5026 return Register();
5027}
5028
5029Register fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, Register Op0) {
5030 if (RetVT.SimpleTy != MVT::f32)
5031 return Register();
5032 if ((Subtarget->hasFPARMv8())) {
5033 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
5034 }
5035 return Register();
5036}
5037
5038Register fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, Register Op0) {
5039 if (RetVT.SimpleTy != MVT::f64)
5040 return Register();
5041 if ((Subtarget->hasFPARMv8())) {
5042 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
5043 }
5044 return Register();
5045}
5046
5047Register fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5048 if (RetVT.SimpleTy != MVT::v4f16)
5049 return Register();
5050 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5051 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
5052 }
5053 return Register();
5054}
5055
5056Register fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5057 if (RetVT.SimpleTy != MVT::v8f16)
5058 return Register();
5059 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5060 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
5061 }
5062 return Register();
5063}
5064
5065Register fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5066 if (RetVT.SimpleTy != MVT::v2f32)
5067 return Register();
5068 if ((Subtarget->isNeonAvailable())) {
5069 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
5070 }
5071 return Register();
5072}
5073
5074Register fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5075 if (RetVT.SimpleTy != MVT::v4f32)
5076 return Register();
5077 if ((Subtarget->isNeonAvailable())) {
5078 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
5079 }
5080 return Register();
5081}
5082
5083Register fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5084 if (RetVT.SimpleTy != MVT::v2f64)
5085 return Register();
5086 if ((Subtarget->isNeonAvailable())) {
5087 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5088 }
5089 return Register();
5090}
5091
5092Register fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, Register Op0) {
5093 switch (VT.SimpleTy) {
5094 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5095 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5096 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5097 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5098 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5099 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5100 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5101 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5102 default: return Register();
5103 }
5104}
5105
5106// FastEmit functions for ISD::STRICT_FROUND.
5107
5108Register fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5109 if (RetVT.SimpleTy != MVT::f16)
5110 return Register();
5111 if ((Subtarget->hasFullFP16())) {
5112 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5113 }
5114 return Register();
5115}
5116
5117Register fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5118 if (RetVT.SimpleTy != MVT::f32)
5119 return Register();
5120 if ((Subtarget->hasFPARMv8())) {
5121 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5122 }
5123 return Register();
5124}
5125
5126Register fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5127 if (RetVT.SimpleTy != MVT::f64)
5128 return Register();
5129 if ((Subtarget->hasFPARMv8())) {
5130 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5131 }
5132 return Register();
5133}
5134
5135Register fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, Register Op0) {
5136 if (RetVT.SimpleTy != MVT::v4f16)
5137 return Register();
5138 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5139 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5140 }
5141 return Register();
5142}
5143
5144Register fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, Register Op0) {
5145 if (RetVT.SimpleTy != MVT::v8f16)
5146 return Register();
5147 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5148 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5149 }
5150 return Register();
5151}
5152
5153Register fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, Register Op0) {
5154 if (RetVT.SimpleTy != MVT::v2f32)
5155 return Register();
5156 if ((Subtarget->isNeonAvailable())) {
5157 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5158 }
5159 return Register();
5160}
5161
5162Register fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, Register Op0) {
5163 if (RetVT.SimpleTy != MVT::v4f32)
5164 return Register();
5165 if ((Subtarget->isNeonAvailable())) {
5166 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5167 }
5168 return Register();
5169}
5170
5171Register fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, Register Op0) {
5172 if (RetVT.SimpleTy != MVT::v2f64)
5173 return Register();
5174 if ((Subtarget->isNeonAvailable())) {
5175 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5176 }
5177 return Register();
5178}
5179
5180Register fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, Register Op0) {
5181 switch (VT.SimpleTy) {
5182 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5183 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5184 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5185 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5186 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5187 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5188 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5189 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5190 default: return Register();
5191 }
5192}
5193
5194// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5195
5196Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, Register Op0) {
5197 if (RetVT.SimpleTy != MVT::f16)
5198 return Register();
5199 if ((Subtarget->hasFullFP16())) {
5200 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5201 }
5202 return Register();
5203}
5204
5205Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, Register Op0) {
5206 if (RetVT.SimpleTy != MVT::f32)
5207 return Register();
5208 if ((Subtarget->hasFPARMv8())) {
5209 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5210 }
5211 return Register();
5212}
5213
5214Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, Register Op0) {
5215 if (RetVT.SimpleTy != MVT::f64)
5216 return Register();
5217 if ((Subtarget->hasFPARMv8())) {
5218 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5219 }
5220 return Register();
5221}
5222
5223Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, Register Op0) {
5224 if (RetVT.SimpleTy != MVT::v4f16)
5225 return Register();
5226 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5227 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5228 }
5229 return Register();
5230}
5231
5232Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, Register Op0) {
5233 if (RetVT.SimpleTy != MVT::v8f16)
5234 return Register();
5235 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5236 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5237 }
5238 return Register();
5239}
5240
5241Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, Register Op0) {
5242 if (RetVT.SimpleTy != MVT::v2f32)
5243 return Register();
5244 if ((Subtarget->isNeonAvailable())) {
5245 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5246 }
5247 return Register();
5248}
5249
5250Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, Register Op0) {
5251 if (RetVT.SimpleTy != MVT::v4f32)
5252 return Register();
5253 if ((Subtarget->isNeonAvailable())) {
5254 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5255 }
5256 return Register();
5257}
5258
5259Register fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, Register Op0) {
5260 if (RetVT.SimpleTy != MVT::v2f64)
5261 return Register();
5262 if ((Subtarget->isNeonAvailable())) {
5263 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5264 }
5265 return Register();
5266}
5267
5268Register fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, Register Op0) {
5269 switch (VT.SimpleTy) {
5270 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5271 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5272 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5273 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5274 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5275 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5276 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5277 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5278 default: return Register();
5279 }
5280}
5281
5282// FastEmit functions for ISD::STRICT_FSQRT.
5283
5284Register fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, Register Op0) {
5285 if (RetVT.SimpleTy != MVT::f16)
5286 return Register();
5287 if ((Subtarget->hasFullFP16())) {
5288 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5289 }
5290 return Register();
5291}
5292
5293Register fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, Register Op0) {
5294 if (RetVT.SimpleTy != MVT::f32)
5295 return Register();
5296 if ((Subtarget->hasFPARMv8())) {
5297 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5298 }
5299 return Register();
5300}
5301
5302Register fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, Register Op0) {
5303 if (RetVT.SimpleTy != MVT::f64)
5304 return Register();
5305 if ((Subtarget->hasFPARMv8())) {
5306 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5307 }
5308 return Register();
5309}
5310
5311Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, Register Op0) {
5312 if (RetVT.SimpleTy != MVT::v4f16)
5313 return Register();
5314 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5315 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5316 }
5317 return Register();
5318}
5319
5320Register fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, Register Op0) {
5321 if (RetVT.SimpleTy != MVT::v8f16)
5322 return Register();
5323 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5324 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5325 }
5326 return Register();
5327}
5328
5329Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, Register Op0) {
5330 if (RetVT.SimpleTy != MVT::v2f32)
5331 return Register();
5332 if ((Subtarget->isNeonAvailable())) {
5333 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5334 }
5335 return Register();
5336}
5337
5338Register fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, Register Op0) {
5339 if (RetVT.SimpleTy != MVT::v4f32)
5340 return Register();
5341 if ((Subtarget->isNeonAvailable())) {
5342 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5343 }
5344 return Register();
5345}
5346
5347Register fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, Register Op0) {
5348 if (RetVT.SimpleTy != MVT::v2f64)
5349 return Register();
5350 if ((Subtarget->isNeonAvailable())) {
5351 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5352 }
5353 return Register();
5354}
5355
5356Register fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, Register Op0) {
5357 switch (VT.SimpleTy) {
5358 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5359 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5360 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5361 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5362 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5363 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5364 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5365 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5366 default: return Register();
5367 }
5368}
5369
5370// FastEmit functions for ISD::STRICT_FTRUNC.
5371
5372Register fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, Register Op0) {
5373 if (RetVT.SimpleTy != MVT::f16)
5374 return Register();
5375 if ((Subtarget->hasFullFP16())) {
5376 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5377 }
5378 return Register();
5379}
5380
5381Register fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, Register Op0) {
5382 if (RetVT.SimpleTy != MVT::f32)
5383 return Register();
5384 if ((Subtarget->hasFPARMv8())) {
5385 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5386 }
5387 return Register();
5388}
5389
5390Register fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, Register Op0) {
5391 if (RetVT.SimpleTy != MVT::f64)
5392 return Register();
5393 if ((Subtarget->hasFPARMv8())) {
5394 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5395 }
5396 return Register();
5397}
5398
5399Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, Register Op0) {
5400 if (RetVT.SimpleTy != MVT::v4f16)
5401 return Register();
5402 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5403 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5404 }
5405 return Register();
5406}
5407
5408Register fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, Register Op0) {
5409 if (RetVT.SimpleTy != MVT::v8f16)
5410 return Register();
5411 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5412 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5413 }
5414 return Register();
5415}
5416
5417Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, Register Op0) {
5418 if (RetVT.SimpleTy != MVT::v2f32)
5419 return Register();
5420 if ((Subtarget->isNeonAvailable())) {
5421 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5422 }
5423 return Register();
5424}
5425
5426Register fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, Register Op0) {
5427 if (RetVT.SimpleTy != MVT::v4f32)
5428 return Register();
5429 if ((Subtarget->isNeonAvailable())) {
5430 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5431 }
5432 return Register();
5433}
5434
5435Register fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, Register Op0) {
5436 if (RetVT.SimpleTy != MVT::v2f64)
5437 return Register();
5438 if ((Subtarget->isNeonAvailable())) {
5439 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5440 }
5441 return Register();
5442}
5443
5444Register fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, Register Op0) {
5445 switch (VT.SimpleTy) {
5446 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5447 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5448 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5449 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5450 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5451 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5452 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5453 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5454 default: return Register();
5455 }
5456}
5457
5458// FastEmit functions for ISD::STRICT_LLROUND.
5459
5460Register fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5461 if (RetVT.SimpleTy != MVT::i64)
5462 return Register();
5463 if ((Subtarget->hasFullFP16())) {
5464 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5465 }
5466 return Register();
5467}
5468
5469Register fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5470 if (RetVT.SimpleTy != MVT::i64)
5471 return Register();
5472 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5473}
5474
5475Register fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5476 if (RetVT.SimpleTy != MVT::i64)
5477 return Register();
5478 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5479}
5480
5481Register fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, Register Op0) {
5482 switch (VT.SimpleTy) {
5483 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5484 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5485 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5486 default: return Register();
5487 }
5488}
5489
5490// FastEmit functions for ISD::STRICT_LROUND.
5491
5492Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Register Op0) {
5493 if ((Subtarget->hasFullFP16())) {
5494 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5495 }
5496 return Register();
5497}
5498
5499Register fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Register Op0) {
5500 if ((Subtarget->hasFullFP16())) {
5501 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5502 }
5503 return Register();
5504}
5505
5506Register fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, Register Op0) {
5507switch (RetVT.SimpleTy) {
5508 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5509 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5510 default: return Register();
5511}
5512}
5513
5514Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Register Op0) {
5515 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5516}
5517
5518Register fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Register Op0) {
5519 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5520}
5521
5522Register fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, Register Op0) {
5523switch (RetVT.SimpleTy) {
5524 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
5525 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
5526 default: return Register();
5527}
5528}
5529
5530Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Register Op0) {
5531 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
5532}
5533
5534Register fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Register Op0) {
5535 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5536}
5537
5538Register fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, Register Op0) {
5539switch (RetVT.SimpleTy) {
5540 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
5541 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
5542 default: return Register();
5543}
5544}
5545
5546Register fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, Register Op0) {
5547 switch (VT.SimpleTy) {
5548 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
5549 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
5550 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
5551 default: return Register();
5552 }
5553}
5554
5555// FastEmit functions for ISD::STRICT_SINT_TO_FP.
5556
5557Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5558 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5559 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5560 }
5561 if ((Subtarget->hasFullFP16())) {
5562 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5563 }
5564 return Register();
5565}
5566
5567Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5568 if ((Subtarget->hasFPARMv8())) {
5569 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5570 }
5571 return Register();
5572}
5573
5574Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5575 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5576 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5577 }
5578 if ((Subtarget->hasFPARMv8())) {
5579 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5580 }
5581 return Register();
5582}
5583
5584Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5585switch (RetVT.SimpleTy) {
5586 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5587 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5588 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5589 default: return Register();
5590}
5591}
5592
5593Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5594 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5595 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5596 }
5597 if ((Subtarget->hasFullFP16())) {
5598 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5599 }
5600 return Register();
5601}
5602
5603Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5604 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5605 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5606 }
5607 if ((Subtarget->hasFPARMv8())) {
5608 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5609 }
5610 return Register();
5611}
5612
5613Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5614 if ((Subtarget->hasFPARMv8())) {
5615 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5616 }
5617 return Register();
5618}
5619
5620Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5621switch (RetVT.SimpleTy) {
5622 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5623 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5624 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5625 default: return Register();
5626}
5627}
5628
5629Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5630 if (RetVT.SimpleTy != MVT::v4f16)
5631 return Register();
5632 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5633 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5634 }
5635 return Register();
5636}
5637
5638Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5639 if (RetVT.SimpleTy != MVT::v8f16)
5640 return Register();
5641 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5642 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5643 }
5644 return Register();
5645}
5646
5647Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5648 if (RetVT.SimpleTy != MVT::v2f32)
5649 return Register();
5650 if ((Subtarget->isNeonAvailable())) {
5651 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5652 }
5653 return Register();
5654}
5655
5656Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5657 if (RetVT.SimpleTy != MVT::v4f32)
5658 return Register();
5659 if ((Subtarget->isNeonAvailable())) {
5660 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5661 }
5662 return Register();
5663}
5664
5665Register fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5666 if (RetVT.SimpleTy != MVT::v2f64)
5667 return Register();
5668 if ((Subtarget->isNeonAvailable())) {
5669 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5670 }
5671 return Register();
5672}
5673
5674Register fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5675 switch (VT.SimpleTy) {
5676 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
5677 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
5678 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5679 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5680 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5681 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5682 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5683 default: return Register();
5684 }
5685}
5686
5687// FastEmit functions for ISD::STRICT_UINT_TO_FP.
5688
5689Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5690 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5691 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5692 }
5693 if ((Subtarget->hasFullFP16())) {
5694 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5695 }
5696 return Register();
5697}
5698
5699Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5700 if ((Subtarget->hasFPARMv8())) {
5701 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5702 }
5703 return Register();
5704}
5705
5706Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5707 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5708 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
5709 }
5710 if ((Subtarget->hasFPARMv8())) {
5711 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
5712 }
5713 return Register();
5714}
5715
5716Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
5717switch (RetVT.SimpleTy) {
5718 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
5719 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
5720 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
5721 default: return Register();
5722}
5723}
5724
5725Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
5726 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5727 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
5728 }
5729 if ((Subtarget->hasFullFP16())) {
5730 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
5731 }
5732 return Register();
5733}
5734
5735Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
5736 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5737 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
5738 }
5739 if ((Subtarget->hasFPARMv8())) {
5740 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
5741 }
5742 return Register();
5743}
5744
5745Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
5746 if ((Subtarget->hasFPARMv8())) {
5747 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
5748 }
5749 return Register();
5750}
5751
5752Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
5753switch (RetVT.SimpleTy) {
5754 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
5755 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
5756 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
5757 default: return Register();
5758}
5759}
5760
5761Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
5762 if (RetVT.SimpleTy != MVT::v4f16)
5763 return Register();
5764 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5765 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
5766 }
5767 return Register();
5768}
5769
5770Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
5771 if (RetVT.SimpleTy != MVT::v8f16)
5772 return Register();
5773 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5774 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
5775 }
5776 return Register();
5777}
5778
5779Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
5780 if (RetVT.SimpleTy != MVT::v2f32)
5781 return Register();
5782 if ((Subtarget->isNeonAvailable())) {
5783 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
5784 }
5785 return Register();
5786}
5787
5788Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
5789 if (RetVT.SimpleTy != MVT::v4f32)
5790 return Register();
5791 if ((Subtarget->isNeonAvailable())) {
5792 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
5793 }
5794 return Register();
5795}
5796
5797Register fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
5798 if (RetVT.SimpleTy != MVT::v2f64)
5799 return Register();
5800 if ((Subtarget->isNeonAvailable())) {
5801 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
5802 }
5803 return Register();
5804}
5805
5806Register fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
5807 switch (VT.SimpleTy) {
5808 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
5809 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
5810 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
5811 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
5812 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
5813 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
5814 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
5815 default: return Register();
5816 }
5817}
5818
5819// FastEmit functions for ISD::TRUNCATE.
5820
5821Register fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, Register Op0) {
5822 if (RetVT.SimpleTy != MVT::i32)
5823 return Register();
5824 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
5825}
5826
5827Register fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, Register Op0) {
5828 if (RetVT.SimpleTy != MVT::v8i8)
5829 return Register();
5830 if ((Subtarget->isNeonAvailable())) {
5831 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5832 }
5833 return Register();
5834}
5835
5836Register fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, Register Op0) {
5837 if (RetVT.SimpleTy != MVT::v4i16)
5838 return Register();
5839 if ((Subtarget->isNeonAvailable())) {
5840 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5841 }
5842 return Register();
5843}
5844
5845Register fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, Register Op0) {
5846 if (RetVT.SimpleTy != MVT::v2i32)
5847 return Register();
5848 if ((Subtarget->isNeonAvailable())) {
5849 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5850 }
5851 return Register();
5852}
5853
5854Register fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, Register Op0) {
5855 switch (VT.SimpleTy) {
5856 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
5857 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
5858 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
5859 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
5860 default: return Register();
5861 }
5862}
5863
5864// FastEmit functions for ISD::TRUNCATE_SSAT_S.
5865
5866Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(MVT RetVT, Register Op0) {
5867 if (RetVT.SimpleTy != MVT::v8i8)
5868 return Register();
5869 if ((Subtarget->isNeonAvailable())) {
5870 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5871 }
5872 return Register();
5873}
5874
5875Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(MVT RetVT, Register Op0) {
5876 if (RetVT.SimpleTy != MVT::v4i16)
5877 return Register();
5878 if ((Subtarget->isNeonAvailable())) {
5879 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5880 }
5881 return Register();
5882}
5883
5884Register fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(MVT RetVT, Register Op0) {
5885 if (RetVT.SimpleTy != MVT::v2i32)
5886 return Register();
5887 if ((Subtarget->isNeonAvailable())) {
5888 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5889 }
5890 return Register();
5891}
5892
5893Register fastEmit_ISD_TRUNCATE_SSAT_S_r(MVT VT, MVT RetVT, Register Op0) {
5894 switch (VT.SimpleTy) {
5895 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v8i16_r(RetVT, Op0);
5896 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v4i32_r(RetVT, Op0);
5897 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_S_MVT_v2i64_r(RetVT, Op0);
5898 default: return Register();
5899 }
5900}
5901
5902// FastEmit functions for ISD::TRUNCATE_SSAT_U.
5903
5904Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5905 if (RetVT.SimpleTy != MVT::v8i8)
5906 return Register();
5907 if ((Subtarget->isNeonAvailable())) {
5908 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5909 }
5910 return Register();
5911}
5912
5913Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5914 if (RetVT.SimpleTy != MVT::v4i16)
5915 return Register();
5916 if ((Subtarget->isNeonAvailable())) {
5917 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5918 }
5919 return Register();
5920}
5921
5922Register fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5923 if (RetVT.SimpleTy != MVT::v2i32)
5924 return Register();
5925 if ((Subtarget->isNeonAvailable())) {
5926 return fastEmitInst_r(MachineInstOpcode: AArch64::SQXTUNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5927 }
5928 return Register();
5929}
5930
5931Register fastEmit_ISD_TRUNCATE_SSAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5932 switch (VT.SimpleTy) {
5933 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v8i16_r(RetVT, Op0);
5934 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v4i32_r(RetVT, Op0);
5935 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_SSAT_U_MVT_v2i64_r(RetVT, Op0);
5936 default: return Register();
5937 }
5938}
5939
5940// FastEmit functions for ISD::TRUNCATE_USAT_U.
5941
5942Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(MVT RetVT, Register Op0) {
5943 if (RetVT.SimpleTy != MVT::v8i8)
5944 return Register();
5945 if ((Subtarget->isNeonAvailable())) {
5946 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
5947 }
5948 return Register();
5949}
5950
5951Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(MVT RetVT, Register Op0) {
5952 if (RetVT.SimpleTy != MVT::v4i16)
5953 return Register();
5954 if ((Subtarget->isNeonAvailable())) {
5955 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5956 }
5957 return Register();
5958}
5959
5960Register fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(MVT RetVT, Register Op0) {
5961 if (RetVT.SimpleTy != MVT::v2i32)
5962 return Register();
5963 if ((Subtarget->isNeonAvailable())) {
5964 return fastEmitInst_r(MachineInstOpcode: AArch64::UQXTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5965 }
5966 return Register();
5967}
5968
5969Register fastEmit_ISD_TRUNCATE_USAT_U_r(MVT VT, MVT RetVT, Register Op0) {
5970 switch (VT.SimpleTy) {
5971 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v8i16_r(RetVT, Op0);
5972 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v4i32_r(RetVT, Op0);
5973 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_USAT_U_MVT_v2i64_r(RetVT, Op0);
5974 default: return Register();
5975 }
5976}
5977
5978// FastEmit functions for ISD::UINT_TO_FP.
5979
5980Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Register Op0) {
5981 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5982 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHSr, RC: &AArch64::FPR16RegClass, Op0);
5983 }
5984 if ((Subtarget->hasFullFP16())) {
5985 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
5986 }
5987 return Register();
5988}
5989
5990Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Register Op0) {
5991 if ((Subtarget->hasFPARMv8())) {
5992 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
5993 }
5994 return Register();
5995}
5996
5997Register fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Register Op0) {
5998 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
5999 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFDSr, RC: &AArch64::FPR64RegClass, Op0);
6000 }
6001 if ((Subtarget->hasFPARMv8())) {
6002 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6003 }
6004 return Register();
6005}
6006
6007Register fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, Register Op0) {
6008switch (RetVT.SimpleTy) {
6009 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6010 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6011 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6012 default: return Register();
6013}
6014}
6015
6016Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Register Op0) {
6017 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
6018 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFHDr, RC: &AArch64::FPR16RegClass, Op0);
6019 }
6020 if ((Subtarget->hasFullFP16())) {
6021 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6022 }
6023 return Register();
6024}
6025
6026Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Register Op0) {
6027 if ((Subtarget->hasFPRCVT()) && (Subtarget->isNeonAvailable())) {
6028 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFSDr, RC: &AArch64::FPR32RegClass, Op0);
6029 }
6030 if ((Subtarget->hasFPARMv8())) {
6031 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6032 }
6033 return Register();
6034}
6035
6036Register fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Register Op0) {
6037 if ((Subtarget->hasFPARMv8())) {
6038 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6039 }
6040 return Register();
6041}
6042
6043Register fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, Register Op0) {
6044switch (RetVT.SimpleTy) {
6045 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6046 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6047 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6048 default: return Register();
6049}
6050}
6051
6052Register fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, Register Op0) {
6053 if (RetVT.SimpleTy != MVT::v4f16)
6054 return Register();
6055 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6056 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6057 }
6058 return Register();
6059}
6060
6061Register fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, Register Op0) {
6062 if (RetVT.SimpleTy != MVT::v8f16)
6063 return Register();
6064 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6065 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6066 }
6067 return Register();
6068}
6069
6070Register fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, Register Op0) {
6071 if (RetVT.SimpleTy != MVT::v2f32)
6072 return Register();
6073 if ((Subtarget->isNeonAvailable())) {
6074 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6075 }
6076 return Register();
6077}
6078
6079Register fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, Register Op0) {
6080 if (RetVT.SimpleTy != MVT::v4f32)
6081 return Register();
6082 if ((Subtarget->isNeonAvailable())) {
6083 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6084 }
6085 return Register();
6086}
6087
6088Register fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, Register Op0) {
6089 if (RetVT.SimpleTy != MVT::v2f64)
6090 return Register();
6091 if ((Subtarget->isNeonAvailable())) {
6092 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6093 }
6094 return Register();
6095}
6096
6097Register fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, Register Op0) {
6098 switch (VT.SimpleTy) {
6099 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6100 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6101 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6102 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6103 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6104 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6105 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6106 default: return Register();
6107 }
6108}
6109
6110// FastEmit functions for ISD::VECREDUCE_ADD.
6111
6112Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, Register Op0) {
6113 if (RetVT.SimpleTy != MVT::i8)
6114 return Register();
6115 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6116}
6117
6118Register fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, Register Op0) {
6119 if (RetVT.SimpleTy != MVT::i8)
6120 return Register();
6121 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6122}
6123
6124Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, Register Op0) {
6125 if (RetVT.SimpleTy != MVT::i16)
6126 return Register();
6127 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6128}
6129
6130Register fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, Register Op0) {
6131 if (RetVT.SimpleTy != MVT::i16)
6132 return Register();
6133 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6134}
6135
6136Register fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, Register Op0) {
6137 if (RetVT.SimpleTy != MVT::i32)
6138 return Register();
6139 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6140}
6141
6142Register fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, Register Op0) {
6143 if (RetVT.SimpleTy != MVT::i64)
6144 return Register();
6145 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6146}
6147
6148Register fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, Register Op0) {
6149 switch (VT.SimpleTy) {
6150 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6151 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6152 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6153 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6154 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6155 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6156 default: return Register();
6157 }
6158}
6159
6160// FastEmit functions for ISD::VECREDUCE_FADD.
6161
6162Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, Register Op0) {
6163 if (RetVT.SimpleTy != MVT::f32)
6164 return Register();
6165 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6166}
6167
6168Register fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, Register Op0) {
6169 if (RetVT.SimpleTy != MVT::f64)
6170 return Register();
6171 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6172}
6173
6174Register fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, Register Op0) {
6175 switch (VT.SimpleTy) {
6176 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6177 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6178 default: return Register();
6179 }
6180}
6181
6182// FastEmit functions for ISD::VECREDUCE_FMAX.
6183
6184Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, Register Op0) {
6185 if (RetVT.SimpleTy != MVT::f16)
6186 return Register();
6187 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6188 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6189 }
6190 return Register();
6191}
6192
6193Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, Register Op0) {
6194 if (RetVT.SimpleTy != MVT::f16)
6195 return Register();
6196 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6197 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6198 }
6199 return Register();
6200}
6201
6202Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, Register Op0) {
6203 if (RetVT.SimpleTy != MVT::f32)
6204 return Register();
6205 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6206}
6207
6208Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, Register Op0) {
6209 if (RetVT.SimpleTy != MVT::f32)
6210 return Register();
6211 if ((Subtarget->isNeonAvailable())) {
6212 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6213 }
6214 return Register();
6215}
6216
6217Register fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, Register Op0) {
6218 if (RetVT.SimpleTy != MVT::f64)
6219 return Register();
6220 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6221}
6222
6223Register fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, Register Op0) {
6224 switch (VT.SimpleTy) {
6225 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6226 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6227 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6228 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6229 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6230 default: return Register();
6231 }
6232}
6233
6234// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6235
6236Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6237 if (RetVT.SimpleTy != MVT::f16)
6238 return Register();
6239 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6240 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6241 }
6242 return Register();
6243}
6244
6245Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6246 if (RetVT.SimpleTy != MVT::f16)
6247 return Register();
6248 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6249 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6250 }
6251 return Register();
6252}
6253
6254Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6255 if (RetVT.SimpleTy != MVT::f32)
6256 return Register();
6257 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6258}
6259
6260Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6261 if (RetVT.SimpleTy != MVT::f32)
6262 return Register();
6263 if ((Subtarget->isNeonAvailable())) {
6264 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6265 }
6266 return Register();
6267}
6268
6269Register fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6270 if (RetVT.SimpleTy != MVT::f64)
6271 return Register();
6272 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6273}
6274
6275Register fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6276 switch (VT.SimpleTy) {
6277 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6278 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6279 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6280 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6281 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6282 default: return Register();
6283 }
6284}
6285
6286// FastEmit functions for ISD::VECREDUCE_FMIN.
6287
6288Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, Register Op0) {
6289 if (RetVT.SimpleTy != MVT::f16)
6290 return Register();
6291 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6292 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6293 }
6294 return Register();
6295}
6296
6297Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, Register Op0) {
6298 if (RetVT.SimpleTy != MVT::f16)
6299 return Register();
6300 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6301 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6302 }
6303 return Register();
6304}
6305
6306Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, Register Op0) {
6307 if (RetVT.SimpleTy != MVT::f32)
6308 return Register();
6309 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6310}
6311
6312Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, Register Op0) {
6313 if (RetVT.SimpleTy != MVT::f32)
6314 return Register();
6315 if ((Subtarget->isNeonAvailable())) {
6316 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6317 }
6318 return Register();
6319}
6320
6321Register fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, Register Op0) {
6322 if (RetVT.SimpleTy != MVT::f64)
6323 return Register();
6324 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6325}
6326
6327Register fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, Register Op0) {
6328 switch (VT.SimpleTy) {
6329 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6330 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6331 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6332 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6333 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6334 default: return Register();
6335 }
6336}
6337
6338// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6339
6340Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, Register Op0) {
6341 if (RetVT.SimpleTy != MVT::f16)
6342 return Register();
6343 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6344 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6345 }
6346 return Register();
6347}
6348
6349Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, Register Op0) {
6350 if (RetVT.SimpleTy != MVT::f16)
6351 return Register();
6352 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6353 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6354 }
6355 return Register();
6356}
6357
6358Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, Register Op0) {
6359 if (RetVT.SimpleTy != MVT::f32)
6360 return Register();
6361 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6362}
6363
6364Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, Register Op0) {
6365 if (RetVT.SimpleTy != MVT::f32)
6366 return Register();
6367 if ((Subtarget->isNeonAvailable())) {
6368 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6369 }
6370 return Register();
6371}
6372
6373Register fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, Register Op0) {
6374 if (RetVT.SimpleTy != MVT::f64)
6375 return Register();
6376 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6377}
6378
6379Register fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, Register Op0) {
6380 switch (VT.SimpleTy) {
6381 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6382 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6383 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6384 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6385 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6386 default: return Register();
6387 }
6388}
6389
6390// FastEmit functions for ISD::VECREDUCE_SMAX.
6391
6392Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6393 if (RetVT.SimpleTy != MVT::i8)
6394 return Register();
6395 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6396}
6397
6398Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6399 if (RetVT.SimpleTy != MVT::i8)
6400 return Register();
6401 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6402}
6403
6404Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6405 if (RetVT.SimpleTy != MVT::i16)
6406 return Register();
6407 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6408}
6409
6410Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6411 if (RetVT.SimpleTy != MVT::i16)
6412 return Register();
6413 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6414}
6415
6416Register fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6417 if (RetVT.SimpleTy != MVT::i32)
6418 return Register();
6419 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6420}
6421
6422Register fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, Register Op0) {
6423 switch (VT.SimpleTy) {
6424 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6425 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6426 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6427 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6428 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6429 default: return Register();
6430 }
6431}
6432
6433// FastEmit functions for ISD::VECREDUCE_SMIN.
6434
6435Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6436 if (RetVT.SimpleTy != MVT::i8)
6437 return Register();
6438 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6439}
6440
6441Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6442 if (RetVT.SimpleTy != MVT::i8)
6443 return Register();
6444 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6445}
6446
6447Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6448 if (RetVT.SimpleTy != MVT::i16)
6449 return Register();
6450 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6451}
6452
6453Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6454 if (RetVT.SimpleTy != MVT::i16)
6455 return Register();
6456 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6457}
6458
6459Register fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6460 if (RetVT.SimpleTy != MVT::i32)
6461 return Register();
6462 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6463}
6464
6465Register fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, Register Op0) {
6466 switch (VT.SimpleTy) {
6467 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6468 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6469 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6470 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6471 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6472 default: return Register();
6473 }
6474}
6475
6476// FastEmit functions for ISD::VECREDUCE_UMAX.
6477
6478Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, Register Op0) {
6479 if (RetVT.SimpleTy != MVT::i8)
6480 return Register();
6481 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6482}
6483
6484Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, Register Op0) {
6485 if (RetVT.SimpleTy != MVT::i8)
6486 return Register();
6487 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6488}
6489
6490Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, Register Op0) {
6491 if (RetVT.SimpleTy != MVT::i16)
6492 return Register();
6493 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6494}
6495
6496Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, Register Op0) {
6497 if (RetVT.SimpleTy != MVT::i16)
6498 return Register();
6499 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6500}
6501
6502Register fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, Register Op0) {
6503 if (RetVT.SimpleTy != MVT::i32)
6504 return Register();
6505 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6506}
6507
6508Register fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, Register Op0) {
6509 switch (VT.SimpleTy) {
6510 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6511 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6512 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6513 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6514 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6515 default: return Register();
6516 }
6517}
6518
6519// FastEmit functions for ISD::VECREDUCE_UMIN.
6520
6521Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, Register Op0) {
6522 if (RetVT.SimpleTy != MVT::i8)
6523 return Register();
6524 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6525}
6526
6527Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, Register Op0) {
6528 if (RetVT.SimpleTy != MVT::i8)
6529 return Register();
6530 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6531}
6532
6533Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, Register Op0) {
6534 if (RetVT.SimpleTy != MVT::i16)
6535 return Register();
6536 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6537}
6538
6539Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, Register Op0) {
6540 if (RetVT.SimpleTy != MVT::i16)
6541 return Register();
6542 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6543}
6544
6545Register fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, Register Op0) {
6546 if (RetVT.SimpleTy != MVT::i32)
6547 return Register();
6548 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6549}
6550
6551Register fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, Register Op0) {
6552 switch (VT.SimpleTy) {
6553 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6554 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6555 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6556 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6557 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6558 default: return Register();
6559 }
6560}
6561
6562// FastEmit functions for ISD::VECTOR_REVERSE.
6563
6564Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, Register Op0) {
6565 if (RetVT.SimpleTy != MVT::nxv2i1)
6566 return Register();
6567 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6568 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6569 }
6570 return Register();
6571}
6572
6573Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, Register Op0) {
6574 if (RetVT.SimpleTy != MVT::nxv4i1)
6575 return Register();
6576 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6577 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6578 }
6579 return Register();
6580}
6581
6582Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, Register Op0) {
6583 if (RetVT.SimpleTy != MVT::nxv8i1)
6584 return Register();
6585 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6587 }
6588 return Register();
6589}
6590
6591Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, Register Op0) {
6592 if (RetVT.SimpleTy != MVT::nxv16i1)
6593 return Register();
6594 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6595 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6596 }
6597 return Register();
6598}
6599
6600Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, Register Op0) {
6601 if (RetVT.SimpleTy != MVT::nxv16i8)
6602 return Register();
6603 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6604 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6605 }
6606 return Register();
6607}
6608
6609Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, Register Op0) {
6610 if (RetVT.SimpleTy != MVT::nxv8i16)
6611 return Register();
6612 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6613 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6614 }
6615 return Register();
6616}
6617
6618Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, Register Op0) {
6619 if (RetVT.SimpleTy != MVT::nxv4i32)
6620 return Register();
6621 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6622 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6623 }
6624 return Register();
6625}
6626
6627Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, Register Op0) {
6628 if (RetVT.SimpleTy != MVT::nxv2i64)
6629 return Register();
6630 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6631 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6632 }
6633 return Register();
6634}
6635
6636Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, Register Op0) {
6637 if (RetVT.SimpleTy != MVT::nxv2f16)
6638 return Register();
6639 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6640 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6641 }
6642 return Register();
6643}
6644
6645Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, Register Op0) {
6646 if (RetVT.SimpleTy != MVT::nxv4f16)
6647 return Register();
6648 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6649 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6650 }
6651 return Register();
6652}
6653
6654Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, Register Op0) {
6655 if (RetVT.SimpleTy != MVT::nxv8f16)
6656 return Register();
6657 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6658 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6659 }
6660 return Register();
6661}
6662
6663Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, Register Op0) {
6664 if (RetVT.SimpleTy != MVT::nxv2bf16)
6665 return Register();
6666 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6667 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6668 }
6669 return Register();
6670}
6671
6672Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, Register Op0) {
6673 if (RetVT.SimpleTy != MVT::nxv4bf16)
6674 return Register();
6675 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6676 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6677 }
6678 return Register();
6679}
6680
6681Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, Register Op0) {
6682 if (RetVT.SimpleTy != MVT::nxv8bf16)
6683 return Register();
6684 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6685 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6686 }
6687 return Register();
6688}
6689
6690Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, Register Op0) {
6691 if (RetVT.SimpleTy != MVT::nxv2f32)
6692 return Register();
6693 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6694 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6695 }
6696 return Register();
6697}
6698
6699Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, Register Op0) {
6700 if (RetVT.SimpleTy != MVT::nxv4f32)
6701 return Register();
6702 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6703 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6704 }
6705 return Register();
6706}
6707
6708Register fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, Register Op0) {
6709 if (RetVT.SimpleTy != MVT::nxv2f64)
6710 return Register();
6711 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6712 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6713 }
6714 return Register();
6715}
6716
6717Register fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, Register Op0) {
6718 switch (VT.SimpleTy) {
6719 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
6720 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
6721 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
6722 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
6723 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
6724 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
6725 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
6726 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
6727 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
6728 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
6729 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
6730 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
6731 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
6732 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
6733 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
6734 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
6735 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
6736 default: return Register();
6737 }
6738}
6739
6740// Top-level FastEmit function.
6741
6742Register fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, Register Op0) override {
6743 switch (Opcode) {
6744 case AArch64ISD::ALLOCATE_ZA_BUFFER: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(VT, RetVT, Op0);
6745 case AArch64ISD::ALLOC_SME_SAVE_BUFFER: return fastEmit_AArch64ISD_ALLOC_SME_SAVE_BUFFER_r(VT, RetVT, Op0);
6746 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
6747 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
6748 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
6749 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
6750 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
6751 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
6752 case AArch64ISD::INIT_TPIDR2OBJ: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_r(VT, RetVT, Op0);
6753 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
6754 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
6755 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
6756 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
6757 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
6758 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
6759 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
6760 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
6761 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
6762 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
6763 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
6764 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
6765 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
6766 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
6767 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
6768 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
6769 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
6770 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
6771 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
6772 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
6773 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
6774 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
6775 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
6776 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
6777 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
6778 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
6779 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
6780 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
6781 case ISD::FP_TO_SINT_SAT: return fastEmit_ISD_FP_TO_SINT_SAT_r(VT, RetVT, Op0);
6782 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
6783 case ISD::FP_TO_UINT_SAT: return fastEmit_ISD_FP_TO_UINT_SAT_r(VT, RetVT, Op0);
6784 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
6785 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
6786 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
6787 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
6788 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
6789 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
6790 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
6791 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
6792 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
6793 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
6794 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
6795 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
6796 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
6797 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
6798 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
6799 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
6800 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
6801 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
6802 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
6803 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
6804 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
6805 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
6806 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
6807 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
6808 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
6809 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
6810 case ISD::TRUNCATE_SSAT_S: return fastEmit_ISD_TRUNCATE_SSAT_S_r(VT, RetVT, Op0);
6811 case ISD::TRUNCATE_SSAT_U: return fastEmit_ISD_TRUNCATE_SSAT_U_r(VT, RetVT, Op0);
6812 case ISD::TRUNCATE_USAT_U: return fastEmit_ISD_TRUNCATE_USAT_U_r(VT, RetVT, Op0);
6813 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
6814 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
6815 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
6816 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
6817 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
6818 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
6819 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
6820 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
6821 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
6822 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
6823 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
6824 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
6825 default: return Register();
6826 }
6827}
6828
6829// FastEmit functions for AArch64ISD::ADDP.
6830
6831Register fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
6832 if (RetVT.SimpleTy != MVT::v8i8)
6833 return Register();
6834 if ((Subtarget->isNeonAvailable())) {
6835 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
6836 }
6837 return Register();
6838}
6839
6840Register fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6841 if (RetVT.SimpleTy != MVT::v16i8)
6842 return Register();
6843 if ((Subtarget->isNeonAvailable())) {
6844 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
6845 }
6846 return Register();
6847}
6848
6849Register fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
6850 if (RetVT.SimpleTy != MVT::v4i16)
6851 return Register();
6852 if ((Subtarget->isNeonAvailable())) {
6853 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6854 }
6855 return Register();
6856}
6857
6858Register fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6859 if (RetVT.SimpleTy != MVT::v8i16)
6860 return Register();
6861 if ((Subtarget->isNeonAvailable())) {
6862 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6863 }
6864 return Register();
6865}
6866
6867Register fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
6868 if (RetVT.SimpleTy != MVT::v2i32)
6869 return Register();
6870 if ((Subtarget->isNeonAvailable())) {
6871 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6872 }
6873 return Register();
6874}
6875
6876Register fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6877 if (RetVT.SimpleTy != MVT::v4i32)
6878 return Register();
6879 if ((Subtarget->isNeonAvailable())) {
6880 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6881 }
6882 return Register();
6883}
6884
6885Register fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6886 if (RetVT.SimpleTy != MVT::v2i64)
6887 return Register();
6888 if ((Subtarget->isNeonAvailable())) {
6889 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6890 }
6891 return Register();
6892}
6893
6894Register fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
6895 if (RetVT.SimpleTy != MVT::v4f16)
6896 return Register();
6897 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6898 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
6899 }
6900 return Register();
6901}
6902
6903Register fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
6904 if (RetVT.SimpleTy != MVT::v8f16)
6905 return Register();
6906 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6907 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
6908 }
6909 return Register();
6910}
6911
6912Register fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
6913 if (RetVT.SimpleTy != MVT::v2f32)
6914 return Register();
6915 if ((Subtarget->isNeonAvailable())) {
6916 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
6917 }
6918 return Register();
6919}
6920
6921Register fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
6922 if (RetVT.SimpleTy != MVT::v4f32)
6923 return Register();
6924 if ((Subtarget->isNeonAvailable())) {
6925 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
6926 }
6927 return Register();
6928}
6929
6930Register fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
6931 if (RetVT.SimpleTy != MVT::v2f64)
6932 return Register();
6933 if ((Subtarget->isNeonAvailable())) {
6934 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
6935 }
6936 return Register();
6937}
6938
6939Register fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6940 switch (VT.SimpleTy) {
6941 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
6942 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
6943 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
6944 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
6945 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
6946 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
6947 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
6948 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
6949 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
6950 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
6951 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
6952 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
6953 default: return Register();
6954 }
6955}
6956
6957// FastEmit functions for AArch64ISD::BIC.
6958
6959Register fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
6960 if (RetVT.SimpleTy != MVT::nxv16i8)
6961 return Register();
6962 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6963 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6964 }
6965 return Register();
6966}
6967
6968Register fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
6969 if (RetVT.SimpleTy != MVT::nxv8i16)
6970 return Register();
6971 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6972 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6973 }
6974 return Register();
6975}
6976
6977Register fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
6978 if (RetVT.SimpleTy != MVT::nxv4i32)
6979 return Register();
6980 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6981 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6982 }
6983 return Register();
6984}
6985
6986Register fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
6987 if (RetVT.SimpleTy != MVT::nxv2i64)
6988 return Register();
6989 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
6990 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
6991 }
6992 return Register();
6993}
6994
6995Register fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
6996 switch (VT.SimpleTy) {
6997 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
6998 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
6999 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7000 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7001 default: return Register();
7002 }
7003}
7004
7005// FastEmit functions for AArch64ISD::FCMEQ.
7006
7007Register fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7008 if (RetVT.SimpleTy != MVT::i32)
7009 return Register();
7010 if ((Subtarget->isNeonAvailable())) {
7011 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7012 }
7013 return Register();
7014}
7015
7016Register fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7017 if (RetVT.SimpleTy != MVT::i64)
7018 return Register();
7019 if ((Subtarget->isNeonAvailable())) {
7020 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7021 }
7022 return Register();
7023}
7024
7025Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7026 if (RetVT.SimpleTy != MVT::v4i16)
7027 return Register();
7028 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7029 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7030 }
7031 return Register();
7032}
7033
7034Register fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7035 if (RetVT.SimpleTy != MVT::v8i16)
7036 return Register();
7037 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7038 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7039 }
7040 return Register();
7041}
7042
7043Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7044 if (RetVT.SimpleTy != MVT::v2i32)
7045 return Register();
7046 if ((Subtarget->isNeonAvailable())) {
7047 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7048 }
7049 return Register();
7050}
7051
7052Register fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7053 if (RetVT.SimpleTy != MVT::v4i32)
7054 return Register();
7055 if ((Subtarget->isNeonAvailable())) {
7056 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7057 }
7058 return Register();
7059}
7060
7061Register fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7062 if (RetVT.SimpleTy != MVT::v1i64)
7063 return Register();
7064 if ((Subtarget->isNeonAvailable())) {
7065 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7066 }
7067 return Register();
7068}
7069
7070Register fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7071 if (RetVT.SimpleTy != MVT::v2i64)
7072 return Register();
7073 if ((Subtarget->isNeonAvailable())) {
7074 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7075 }
7076 return Register();
7077}
7078
7079Register fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7080 switch (VT.SimpleTy) {
7081 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7082 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7083 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7084 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7085 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7086 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7087 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7088 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7089 default: return Register();
7090 }
7091}
7092
7093// FastEmit functions for AArch64ISD::FCMGE.
7094
7095Register fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7096 if (RetVT.SimpleTy != MVT::i32)
7097 return Register();
7098 if ((Subtarget->isNeonAvailable())) {
7099 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7100 }
7101 return Register();
7102}
7103
7104Register fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7105 if (RetVT.SimpleTy != MVT::i64)
7106 return Register();
7107 if ((Subtarget->isNeonAvailable())) {
7108 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7109 }
7110 return Register();
7111}
7112
7113Register fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7114 if (RetVT.SimpleTy != MVT::v4i16)
7115 return Register();
7116 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7117 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7118 }
7119 return Register();
7120}
7121
7122Register fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7123 if (RetVT.SimpleTy != MVT::v8i16)
7124 return Register();
7125 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7126 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7127 }
7128 return Register();
7129}
7130
7131Register fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7132 if (RetVT.SimpleTy != MVT::v2i32)
7133 return Register();
7134 if ((Subtarget->isNeonAvailable())) {
7135 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7136 }
7137 return Register();
7138}
7139
7140Register fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7141 if (RetVT.SimpleTy != MVT::v4i32)
7142 return Register();
7143 if ((Subtarget->isNeonAvailable())) {
7144 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7145 }
7146 return Register();
7147}
7148
7149Register fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7150 if (RetVT.SimpleTy != MVT::v1i64)
7151 return Register();
7152 if ((Subtarget->isNeonAvailable())) {
7153 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7154 }
7155 return Register();
7156}
7157
7158Register fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7159 if (RetVT.SimpleTy != MVT::v2i64)
7160 return Register();
7161 if ((Subtarget->isNeonAvailable())) {
7162 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7163 }
7164 return Register();
7165}
7166
7167Register fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7168 switch (VT.SimpleTy) {
7169 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7170 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7171 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7172 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7173 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7174 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7175 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7176 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7177 default: return Register();
7178 }
7179}
7180
7181// FastEmit functions for AArch64ISD::FCMGT.
7182
7183Register fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7184 if (RetVT.SimpleTy != MVT::i32)
7185 return Register();
7186 if ((Subtarget->isNeonAvailable())) {
7187 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7188 }
7189 return Register();
7190}
7191
7192Register fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7193 if (RetVT.SimpleTy != MVT::i64)
7194 return Register();
7195 if ((Subtarget->isNeonAvailable())) {
7196 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7197 }
7198 return Register();
7199}
7200
7201Register fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7202 if (RetVT.SimpleTy != MVT::v4i16)
7203 return Register();
7204 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7205 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7206 }
7207 return Register();
7208}
7209
7210Register fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7211 if (RetVT.SimpleTy != MVT::v8i16)
7212 return Register();
7213 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7214 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7215 }
7216 return Register();
7217}
7218
7219Register fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7220 if (RetVT.SimpleTy != MVT::v2i32)
7221 return Register();
7222 if ((Subtarget->isNeonAvailable())) {
7223 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7224 }
7225 return Register();
7226}
7227
7228Register fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7229 if (RetVT.SimpleTy != MVT::v4i32)
7230 return Register();
7231 if ((Subtarget->isNeonAvailable())) {
7232 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7233 }
7234 return Register();
7235}
7236
7237Register fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
7238 if (RetVT.SimpleTy != MVT::v1i64)
7239 return Register();
7240 if ((Subtarget->isNeonAvailable())) {
7241 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7242 }
7243 return Register();
7244}
7245
7246Register fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7247 if (RetVT.SimpleTy != MVT::v2i64)
7248 return Register();
7249 if ((Subtarget->isNeonAvailable())) {
7250 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7251 }
7252 return Register();
7253}
7254
7255Register fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7256 switch (VT.SimpleTy) {
7257 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
7258 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
7259 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
7260 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
7261 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
7262 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
7263 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
7264 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
7265 default: return Register();
7266 }
7267}
7268
7269// FastEmit functions for AArch64ISD::FCMP.
7270
7271Register fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7272 if (RetVT.SimpleTy != MVT::i32)
7273 return Register();
7274 if ((Subtarget->hasFullFP16())) {
7275 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7276 }
7277 return Register();
7278}
7279
7280Register fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7281 if (RetVT.SimpleTy != MVT::i32)
7282 return Register();
7283 if ((Subtarget->hasFPARMv8())) {
7284 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7285 }
7286 return Register();
7287}
7288
7289Register fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7290 if (RetVT.SimpleTy != MVT::i32)
7291 return Register();
7292 if ((Subtarget->hasFPARMv8())) {
7293 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7294 }
7295 return Register();
7296}
7297
7298Register fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7299 switch (VT.SimpleTy) {
7300 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7301 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7302 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7303 default: return Register();
7304 }
7305}
7306
7307// FastEmit functions for AArch64ISD::FRECPS.
7308
7309Register fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7310 if (RetVT.SimpleTy != MVT::f32)
7311 return Register();
7312 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7313}
7314
7315Register fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7316 if (RetVT.SimpleTy != MVT::f64)
7317 return Register();
7318 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7319}
7320
7321Register fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7322 if (RetVT.SimpleTy != MVT::v2f32)
7323 return Register();
7324 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7325}
7326
7327Register fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7328 if (RetVT.SimpleTy != MVT::v4f32)
7329 return Register();
7330 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7331}
7332
7333Register fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7334 if (RetVT.SimpleTy != MVT::v2f64)
7335 return Register();
7336 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7337}
7338
7339Register fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7340 if (RetVT.SimpleTy != MVT::nxv8f16)
7341 return Register();
7342 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7343 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7344 }
7345 return Register();
7346}
7347
7348Register fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7349 if (RetVT.SimpleTy != MVT::nxv4f32)
7350 return Register();
7351 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7353 }
7354 return Register();
7355}
7356
7357Register fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7358 if (RetVT.SimpleTy != MVT::nxv2f64)
7359 return Register();
7360 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7361 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7362 }
7363 return Register();
7364}
7365
7366Register fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7367 switch (VT.SimpleTy) {
7368 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
7369 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
7370 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
7371 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
7372 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
7373 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7374 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7375 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7376 default: return Register();
7377 }
7378}
7379
7380// FastEmit functions for AArch64ISD::FRSQRTS.
7381
7382Register fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7383 if (RetVT.SimpleTy != MVT::f32)
7384 return Register();
7385 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7386}
7387
7388Register fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7389 if (RetVT.SimpleTy != MVT::f64)
7390 return Register();
7391 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7392}
7393
7394Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7395 if (RetVT.SimpleTy != MVT::v2f32)
7396 return Register();
7397 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7398}
7399
7400Register fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7401 if (RetVT.SimpleTy != MVT::v4f32)
7402 return Register();
7403 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7404}
7405
7406Register fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7407 if (RetVT.SimpleTy != MVT::v2f64)
7408 return Register();
7409 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7410}
7411
7412Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7413 if (RetVT.SimpleTy != MVT::nxv8f16)
7414 return Register();
7415 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7416 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7417 }
7418 return Register();
7419}
7420
7421Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7422 if (RetVT.SimpleTy != MVT::nxv4f32)
7423 return Register();
7424 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7425 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7426 }
7427 return Register();
7428}
7429
7430Register fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7431 if (RetVT.SimpleTy != MVT::nxv2f64)
7432 return Register();
7433 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7434 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7435 }
7436 return Register();
7437}
7438
7439Register fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7440 switch (VT.SimpleTy) {
7441 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
7442 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
7443 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
7444 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
7445 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
7446 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7447 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7448 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7449 default: return Register();
7450 }
7451}
7452
7453// FastEmit functions for AArch64ISD::PMULL.
7454
7455Register fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7456 if (RetVT.SimpleTy != MVT::v8i16)
7457 return Register();
7458 if ((Subtarget->isNeonAvailable())) {
7459 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7460 }
7461 return Register();
7462}
7463
7464Register fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
7465 if (RetVT.SimpleTy != MVT::v16i8)
7466 return Register();
7467 if ((Subtarget->hasAES())) {
7468 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7469 }
7470 return Register();
7471}
7472
7473Register fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7474 switch (VT.SimpleTy) {
7475 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7476 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
7477 default: return Register();
7478 }
7479}
7480
7481// FastEmit functions for AArch64ISD::PTEST.
7482
7483Register fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7484 if (RetVT.SimpleTy != MVT::i32)
7485 return Register();
7486 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7487 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
7488 }
7489 return Register();
7490}
7491
7492Register fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7493 switch (VT.SimpleTy) {
7494 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7495 default: return Register();
7496 }
7497}
7498
7499// FastEmit functions for AArch64ISD::PTEST_ANY.
7500
7501Register fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7502 if (RetVT.SimpleTy != MVT::i32)
7503 return Register();
7504 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7505 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
7506 }
7507 return Register();
7508}
7509
7510Register fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7511 switch (VT.SimpleTy) {
7512 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7513 default: return Register();
7514 }
7515}
7516
7517// FastEmit functions for AArch64ISD::SMULL.
7518
7519Register fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7520 if (RetVT.SimpleTy != MVT::v8i16)
7521 return Register();
7522 if ((Subtarget->isNeonAvailable())) {
7523 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7524 }
7525 return Register();
7526}
7527
7528Register fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7529 if (RetVT.SimpleTy != MVT::v4i32)
7530 return Register();
7531 if ((Subtarget->isNeonAvailable())) {
7532 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7533 }
7534 return Register();
7535}
7536
7537Register fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7538 if (RetVT.SimpleTy != MVT::v2i64)
7539 return Register();
7540 if ((Subtarget->isNeonAvailable())) {
7541 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7542 }
7543 return Register();
7544}
7545
7546Register fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7547 switch (VT.SimpleTy) {
7548 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
7549 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
7550 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
7551 default: return Register();
7552 }
7553}
7554
7555// FastEmit functions for AArch64ISD::STRICT_FCMP.
7556
7557Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7558 if (RetVT.SimpleTy != MVT::i32)
7559 return Register();
7560 if ((Subtarget->hasFullFP16())) {
7561 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7562 }
7563 return Register();
7564}
7565
7566Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7567 if (RetVT.SimpleTy != MVT::i32)
7568 return Register();
7569 if ((Subtarget->hasFPARMv8())) {
7570 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7571 }
7572 return Register();
7573}
7574
7575Register fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7576 if (RetVT.SimpleTy != MVT::i32)
7577 return Register();
7578 if ((Subtarget->hasFPARMv8())) {
7579 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7580 }
7581 return Register();
7582}
7583
7584Register fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7585 switch (VT.SimpleTy) {
7586 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
7587 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
7588 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
7589 default: return Register();
7590 }
7591}
7592
7593// FastEmit functions for AArch64ISD::STRICT_FCMPE.
7594
7595Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
7596 if (RetVT.SimpleTy != MVT::i32)
7597 return Register();
7598 if ((Subtarget->hasFullFP16())) {
7599 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
7600 }
7601 return Register();
7602}
7603
7604Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
7605 if (RetVT.SimpleTy != MVT::i32)
7606 return Register();
7607 if ((Subtarget->hasFPARMv8())) {
7608 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
7609 }
7610 return Register();
7611}
7612
7613Register fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
7614 if (RetVT.SimpleTy != MVT::i32)
7615 return Register();
7616 if ((Subtarget->hasFPARMv8())) {
7617 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
7618 }
7619 return Register();
7620}
7621
7622Register fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7623 switch (VT.SimpleTy) {
7624 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
7625 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
7626 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
7627 default: return Register();
7628 }
7629}
7630
7631// FastEmit functions for AArch64ISD::TBL.
7632
7633Register fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7634 if (RetVT.SimpleTy != MVT::nxv16i8)
7635 return Register();
7636 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7637 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7638 }
7639 return Register();
7640}
7641
7642Register fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7643 if (RetVT.SimpleTy != MVT::nxv8i16)
7644 return Register();
7645 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7646 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7647 }
7648 return Register();
7649}
7650
7651Register fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7652 if (RetVT.SimpleTy != MVT::nxv4i32)
7653 return Register();
7654 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7655 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7656 }
7657 return Register();
7658}
7659
7660Register fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7661 if (RetVT.SimpleTy != MVT::nxv2i64)
7662 return Register();
7663 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7664 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7665 }
7666 return Register();
7667}
7668
7669Register fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7670 switch (VT.SimpleTy) {
7671 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7672 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7673 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7674 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7675 default: return Register();
7676 }
7677}
7678
7679// FastEmit functions for AArch64ISD::TRN1.
7680
7681Register fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
7682 if (RetVT.SimpleTy != MVT::v8i8)
7683 return Register();
7684 if ((Subtarget->isNeonAvailable())) {
7685 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7686 }
7687 return Register();
7688}
7689
7690Register fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7691 if (RetVT.SimpleTy != MVT::v16i8)
7692 return Register();
7693 if ((Subtarget->isNeonAvailable())) {
7694 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7695 }
7696 return Register();
7697}
7698
7699Register fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
7700 if (RetVT.SimpleTy != MVT::v4i16)
7701 return Register();
7702 if ((Subtarget->isNeonAvailable())) {
7703 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7704 }
7705 return Register();
7706}
7707
7708Register fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7709 if (RetVT.SimpleTy != MVT::v8i16)
7710 return Register();
7711 if ((Subtarget->isNeonAvailable())) {
7712 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7713 }
7714 return Register();
7715}
7716
7717Register fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
7718 if (RetVT.SimpleTy != MVT::v2i32)
7719 return Register();
7720 if ((Subtarget->isNeonAvailable())) {
7721 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7722 }
7723 return Register();
7724}
7725
7726Register fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7727 if (RetVT.SimpleTy != MVT::v4i32)
7728 return Register();
7729 if ((Subtarget->isNeonAvailable())) {
7730 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7731 }
7732 return Register();
7733}
7734
7735Register fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7736 if (RetVT.SimpleTy != MVT::v2i64)
7737 return Register();
7738 if ((Subtarget->isNeonAvailable())) {
7739 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7740 }
7741 return Register();
7742}
7743
7744Register fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7745 if (RetVT.SimpleTy != MVT::v4f16)
7746 return Register();
7747 if ((Subtarget->isNeonAvailable())) {
7748 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7749 }
7750 return Register();
7751}
7752
7753Register fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7754 if (RetVT.SimpleTy != MVT::v8f16)
7755 return Register();
7756 if ((Subtarget->isNeonAvailable())) {
7757 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7758 }
7759 return Register();
7760}
7761
7762Register fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7763 if (RetVT.SimpleTy != MVT::v4bf16)
7764 return Register();
7765 if ((Subtarget->isNeonAvailable())) {
7766 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7767 }
7768 return Register();
7769}
7770
7771Register fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7772 if (RetVT.SimpleTy != MVT::v8bf16)
7773 return Register();
7774 if ((Subtarget->isNeonAvailable())) {
7775 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7776 }
7777 return Register();
7778}
7779
7780Register fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7781 if (RetVT.SimpleTy != MVT::v2f32)
7782 return Register();
7783 if ((Subtarget->isNeonAvailable())) {
7784 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7785 }
7786 return Register();
7787}
7788
7789Register fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7790 if (RetVT.SimpleTy != MVT::v4f32)
7791 return Register();
7792 if ((Subtarget->isNeonAvailable())) {
7793 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7794 }
7795 return Register();
7796}
7797
7798Register fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7799 if (RetVT.SimpleTy != MVT::v2f64)
7800 return Register();
7801 if ((Subtarget->isNeonAvailable())) {
7802 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7803 }
7804 return Register();
7805}
7806
7807Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
7808 if (RetVT.SimpleTy != MVT::nxv2i1)
7809 return Register();
7810 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7811 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
7812 }
7813 return Register();
7814}
7815
7816Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
7817 if (RetVT.SimpleTy != MVT::nxv4i1)
7818 return Register();
7819 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7820 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
7821 }
7822 return Register();
7823}
7824
7825Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
7826 if (RetVT.SimpleTy != MVT::nxv8i1)
7827 return Register();
7828 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7829 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
7830 }
7831 return Register();
7832}
7833
7834Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
7835 if (RetVT.SimpleTy != MVT::nxv16i1)
7836 return Register();
7837 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7838 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
7839 }
7840 return Register();
7841}
7842
7843Register fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
7844 if (RetVT.SimpleTy != MVT::nxv16i8)
7845 return Register();
7846 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7847 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
7848 }
7849 return Register();
7850}
7851
7852Register fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
7853 if (RetVT.SimpleTy != MVT::nxv8i16)
7854 return Register();
7855 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7856 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7857 }
7858 return Register();
7859}
7860
7861Register fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
7862 if (RetVT.SimpleTy != MVT::nxv4i32)
7863 return Register();
7864 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7865 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7866 }
7867 return Register();
7868}
7869
7870Register fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
7871 if (RetVT.SimpleTy != MVT::nxv2i64)
7872 return Register();
7873 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7874 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7875 }
7876 return Register();
7877}
7878
7879Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
7880 if (RetVT.SimpleTy != MVT::nxv2f16)
7881 return Register();
7882 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7883 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7884 }
7885 return Register();
7886}
7887
7888Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
7889 if (RetVT.SimpleTy != MVT::nxv4f16)
7890 return Register();
7891 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7892 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7893 }
7894 return Register();
7895}
7896
7897Register fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
7898 if (RetVT.SimpleTy != MVT::nxv8f16)
7899 return Register();
7900 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7901 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7902 }
7903 return Register();
7904}
7905
7906Register fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7907 if (RetVT.SimpleTy != MVT::nxv2bf16)
7908 return Register();
7909 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7910 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7911 }
7912 return Register();
7913}
7914
7915Register fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7916 if (RetVT.SimpleTy != MVT::nxv4bf16)
7917 return Register();
7918 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7919 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7920 }
7921 return Register();
7922}
7923
7924Register fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
7925 if (RetVT.SimpleTy != MVT::nxv8bf16)
7926 return Register();
7927 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7928 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
7929 }
7930 return Register();
7931}
7932
7933Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
7934 if (RetVT.SimpleTy != MVT::nxv2f32)
7935 return Register();
7936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7937 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7938 }
7939 return Register();
7940}
7941
7942Register fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
7943 if (RetVT.SimpleTy != MVT::nxv4f32)
7944 return Register();
7945 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7946 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
7947 }
7948 return Register();
7949}
7950
7951Register fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
7952 if (RetVT.SimpleTy != MVT::nxv2f64)
7953 return Register();
7954 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
7955 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
7956 }
7957 return Register();
7958}
7959
7960Register fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
7961 switch (VT.SimpleTy) {
7962 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
7963 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
7964 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
7965 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
7966 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
7967 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
7968 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
7969 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
7970 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
7971 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
7972 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
7973 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
7974 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
7975 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
7976 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
7977 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
7978 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
7979 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
7980 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7981 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7982 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7983 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7984 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
7985 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
7986 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
7987 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
7988 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
7989 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
7990 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
7991 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
7992 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
7993 default: return Register();
7994 }
7995}
7996
7997// FastEmit functions for AArch64ISD::TRN2.
7998
7999Register fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8000 if (RetVT.SimpleTy != MVT::v8i8)
8001 return Register();
8002 if ((Subtarget->isNeonAvailable())) {
8003 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8004 }
8005 return Register();
8006}
8007
8008Register fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8009 if (RetVT.SimpleTy != MVT::v16i8)
8010 return Register();
8011 if ((Subtarget->isNeonAvailable())) {
8012 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8013 }
8014 return Register();
8015}
8016
8017Register fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8018 if (RetVT.SimpleTy != MVT::v4i16)
8019 return Register();
8020 if ((Subtarget->isNeonAvailable())) {
8021 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8022 }
8023 return Register();
8024}
8025
8026Register fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8027 if (RetVT.SimpleTy != MVT::v8i16)
8028 return Register();
8029 if ((Subtarget->isNeonAvailable())) {
8030 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8031 }
8032 return Register();
8033}
8034
8035Register fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8036 if (RetVT.SimpleTy != MVT::v2i32)
8037 return Register();
8038 if ((Subtarget->isNeonAvailable())) {
8039 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8040 }
8041 return Register();
8042}
8043
8044Register fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8045 if (RetVT.SimpleTy != MVT::v4i32)
8046 return Register();
8047 if ((Subtarget->isNeonAvailable())) {
8048 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8049 }
8050 return Register();
8051}
8052
8053Register fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8054 if (RetVT.SimpleTy != MVT::v2i64)
8055 return Register();
8056 if ((Subtarget->isNeonAvailable())) {
8057 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8058 }
8059 return Register();
8060}
8061
8062Register fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8063 if (RetVT.SimpleTy != MVT::v4f16)
8064 return Register();
8065 if ((Subtarget->isNeonAvailable())) {
8066 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8067 }
8068 return Register();
8069}
8070
8071Register fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8072 if (RetVT.SimpleTy != MVT::v8f16)
8073 return Register();
8074 if ((Subtarget->isNeonAvailable())) {
8075 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8076 }
8077 return Register();
8078}
8079
8080Register fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8081 if (RetVT.SimpleTy != MVT::v4bf16)
8082 return Register();
8083 if ((Subtarget->isNeonAvailable())) {
8084 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8085 }
8086 return Register();
8087}
8088
8089Register fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8090 if (RetVT.SimpleTy != MVT::v8bf16)
8091 return Register();
8092 if ((Subtarget->isNeonAvailable())) {
8093 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8094 }
8095 return Register();
8096}
8097
8098Register fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8099 if (RetVT.SimpleTy != MVT::v2f32)
8100 return Register();
8101 if ((Subtarget->isNeonAvailable())) {
8102 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8103 }
8104 return Register();
8105}
8106
8107Register fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8108 if (RetVT.SimpleTy != MVT::v4f32)
8109 return Register();
8110 if ((Subtarget->isNeonAvailable())) {
8111 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8112 }
8113 return Register();
8114}
8115
8116Register fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8117 if (RetVT.SimpleTy != MVT::v2f64)
8118 return Register();
8119 if ((Subtarget->isNeonAvailable())) {
8120 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8121 }
8122 return Register();
8123}
8124
8125Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8126 if (RetVT.SimpleTy != MVT::nxv2i1)
8127 return Register();
8128 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8129 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8130 }
8131 return Register();
8132}
8133
8134Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8135 if (RetVT.SimpleTy != MVT::nxv4i1)
8136 return Register();
8137 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8138 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8139 }
8140 return Register();
8141}
8142
8143Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8144 if (RetVT.SimpleTy != MVT::nxv8i1)
8145 return Register();
8146 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8147 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8148 }
8149 return Register();
8150}
8151
8152Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8153 if (RetVT.SimpleTy != MVT::nxv16i1)
8154 return Register();
8155 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8156 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8157 }
8158 return Register();
8159}
8160
8161Register fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8162 if (RetVT.SimpleTy != MVT::nxv16i8)
8163 return Register();
8164 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8165 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8166 }
8167 return Register();
8168}
8169
8170Register fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8171 if (RetVT.SimpleTy != MVT::nxv8i16)
8172 return Register();
8173 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8174 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8175 }
8176 return Register();
8177}
8178
8179Register fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8180 if (RetVT.SimpleTy != MVT::nxv4i32)
8181 return Register();
8182 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8183 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8184 }
8185 return Register();
8186}
8187
8188Register fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8189 if (RetVT.SimpleTy != MVT::nxv2i64)
8190 return Register();
8191 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8192 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8193 }
8194 return Register();
8195}
8196
8197Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8198 if (RetVT.SimpleTy != MVT::nxv2f16)
8199 return Register();
8200 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8201 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8202 }
8203 return Register();
8204}
8205
8206Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8207 if (RetVT.SimpleTy != MVT::nxv4f16)
8208 return Register();
8209 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8210 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8211 }
8212 return Register();
8213}
8214
8215Register fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8216 if (RetVT.SimpleTy != MVT::nxv8f16)
8217 return Register();
8218 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8219 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8220 }
8221 return Register();
8222}
8223
8224Register fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8225 if (RetVT.SimpleTy != MVT::nxv2bf16)
8226 return Register();
8227 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8228 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8229 }
8230 return Register();
8231}
8232
8233Register fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8234 if (RetVT.SimpleTy != MVT::nxv4bf16)
8235 return Register();
8236 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8237 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8238 }
8239 return Register();
8240}
8241
8242Register fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8243 if (RetVT.SimpleTy != MVT::nxv8bf16)
8244 return Register();
8245 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8246 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8247 }
8248 return Register();
8249}
8250
8251Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8252 if (RetVT.SimpleTy != MVT::nxv2f32)
8253 return Register();
8254 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8255 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8256 }
8257 return Register();
8258}
8259
8260Register fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8261 if (RetVT.SimpleTy != MVT::nxv4f32)
8262 return Register();
8263 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8264 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8265 }
8266 return Register();
8267}
8268
8269Register fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8270 if (RetVT.SimpleTy != MVT::nxv2f64)
8271 return Register();
8272 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8273 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8274 }
8275 return Register();
8276}
8277
8278Register fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8279 switch (VT.SimpleTy) {
8280 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
8281 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
8282 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
8283 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
8284 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
8285 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
8286 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
8287 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
8288 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
8289 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8290 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8291 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
8292 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
8293 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
8294 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8295 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8296 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8297 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8298 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8299 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8300 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8301 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8302 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8303 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8304 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8305 case MVT::nxv2bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8306 case MVT::nxv4bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8307 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8308 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8309 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8310 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8311 default: return Register();
8312 }
8313}
8314
8315// FastEmit functions for AArch64ISD::UMULL.
8316
8317Register fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8318 if (RetVT.SimpleTy != MVT::v8i16)
8319 return Register();
8320 if ((Subtarget->isNeonAvailable())) {
8321 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8322 }
8323 return Register();
8324}
8325
8326Register fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8327 if (RetVT.SimpleTy != MVT::v4i32)
8328 return Register();
8329 if ((Subtarget->isNeonAvailable())) {
8330 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8331 }
8332 return Register();
8333}
8334
8335Register fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8336 if (RetVT.SimpleTy != MVT::v2i64)
8337 return Register();
8338 if ((Subtarget->isNeonAvailable())) {
8339 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8340 }
8341 return Register();
8342}
8343
8344Register fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8345 switch (VT.SimpleTy) {
8346 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8347 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8348 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8349 default: return Register();
8350 }
8351}
8352
8353// FastEmit functions for AArch64ISD::UZP1.
8354
8355Register fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8356 if (RetVT.SimpleTy != MVT::v8i8)
8357 return Register();
8358 if ((Subtarget->isNeonAvailable())) {
8359 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8360 }
8361 return Register();
8362}
8363
8364Register fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8365 if (RetVT.SimpleTy != MVT::v16i8)
8366 return Register();
8367 if ((Subtarget->isNeonAvailable())) {
8368 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8369 }
8370 return Register();
8371}
8372
8373Register fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8374 if (RetVT.SimpleTy != MVT::v4i16)
8375 return Register();
8376 if ((Subtarget->isNeonAvailable())) {
8377 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8378 }
8379 return Register();
8380}
8381
8382Register fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8383 if (RetVT.SimpleTy != MVT::v8i16)
8384 return Register();
8385 if ((Subtarget->isNeonAvailable())) {
8386 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8387 }
8388 return Register();
8389}
8390
8391Register fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8392 if (RetVT.SimpleTy != MVT::v2i32)
8393 return Register();
8394 if ((Subtarget->isNeonAvailable())) {
8395 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8396 }
8397 return Register();
8398}
8399
8400Register fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8401 if (RetVT.SimpleTy != MVT::v4i32)
8402 return Register();
8403 if ((Subtarget->isNeonAvailable())) {
8404 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8405 }
8406 return Register();
8407}
8408
8409Register fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8410 if (RetVT.SimpleTy != MVT::v2i64)
8411 return Register();
8412 if ((Subtarget->isNeonAvailable())) {
8413 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8414 }
8415 return Register();
8416}
8417
8418Register fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8419 if (RetVT.SimpleTy != MVT::v4f16)
8420 return Register();
8421 if ((Subtarget->isNeonAvailable())) {
8422 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8423 }
8424 return Register();
8425}
8426
8427Register fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8428 if (RetVT.SimpleTy != MVT::v8f16)
8429 return Register();
8430 if ((Subtarget->isNeonAvailable())) {
8431 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8432 }
8433 return Register();
8434}
8435
8436Register fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8437 if (RetVT.SimpleTy != MVT::v4bf16)
8438 return Register();
8439 if ((Subtarget->isNeonAvailable())) {
8440 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8441 }
8442 return Register();
8443}
8444
8445Register fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8446 if (RetVT.SimpleTy != MVT::v8bf16)
8447 return Register();
8448 if ((Subtarget->isNeonAvailable())) {
8449 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8450 }
8451 return Register();
8452}
8453
8454Register fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8455 if (RetVT.SimpleTy != MVT::v2f32)
8456 return Register();
8457 if ((Subtarget->isNeonAvailable())) {
8458 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8459 }
8460 return Register();
8461}
8462
8463Register fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8464 if (RetVT.SimpleTy != MVT::v4f32)
8465 return Register();
8466 if ((Subtarget->isNeonAvailable())) {
8467 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8468 }
8469 return Register();
8470}
8471
8472Register fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8473 if (RetVT.SimpleTy != MVT::v2f64)
8474 return Register();
8475 if ((Subtarget->isNeonAvailable())) {
8476 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8477 }
8478 return Register();
8479}
8480
8481Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8482 if (RetVT.SimpleTy != MVT::nxv2i1)
8483 return Register();
8484 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8485 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8486 }
8487 return Register();
8488}
8489
8490Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8491 if (RetVT.SimpleTy != MVT::nxv4i1)
8492 return Register();
8493 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8494 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8495 }
8496 return Register();
8497}
8498
8499Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8500 if (RetVT.SimpleTy != MVT::nxv8i1)
8501 return Register();
8502 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8503 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8504 }
8505 return Register();
8506}
8507
8508Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8509 if (RetVT.SimpleTy != MVT::nxv16i1)
8510 return Register();
8511 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8512 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8513 }
8514 return Register();
8515}
8516
8517Register fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8518 if (RetVT.SimpleTy != MVT::nxv16i8)
8519 return Register();
8520 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8521 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8522 }
8523 return Register();
8524}
8525
8526Register fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8527 if (RetVT.SimpleTy != MVT::nxv8i16)
8528 return Register();
8529 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8530 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8531 }
8532 return Register();
8533}
8534
8535Register fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8536 if (RetVT.SimpleTy != MVT::nxv4i32)
8537 return Register();
8538 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8539 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8540 }
8541 return Register();
8542}
8543
8544Register fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8545 if (RetVT.SimpleTy != MVT::nxv2i64)
8546 return Register();
8547 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8548 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8549 }
8550 return Register();
8551}
8552
8553Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8554 if (RetVT.SimpleTy != MVT::nxv2f16)
8555 return Register();
8556 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8557 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8558 }
8559 return Register();
8560}
8561
8562Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8563 if (RetVT.SimpleTy != MVT::nxv4f16)
8564 return Register();
8565 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8566 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8567 }
8568 return Register();
8569}
8570
8571Register fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8572 if (RetVT.SimpleTy != MVT::nxv8f16)
8573 return Register();
8574 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8575 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8576 }
8577 return Register();
8578}
8579
8580Register fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8581 if (RetVT.SimpleTy != MVT::nxv2bf16)
8582 return Register();
8583 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8584 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8585 }
8586 return Register();
8587}
8588
8589Register fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8590 if (RetVT.SimpleTy != MVT::nxv4bf16)
8591 return Register();
8592 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8593 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8594 }
8595 return Register();
8596}
8597
8598Register fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8599 if (RetVT.SimpleTy != MVT::nxv8bf16)
8600 return Register();
8601 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8602 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8603 }
8604 return Register();
8605}
8606
8607Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8608 if (RetVT.SimpleTy != MVT::nxv2f32)
8609 return Register();
8610 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8611 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8612 }
8613 return Register();
8614}
8615
8616Register fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8617 if (RetVT.SimpleTy != MVT::nxv4f32)
8618 return Register();
8619 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8620 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8621 }
8622 return Register();
8623}
8624
8625Register fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8626 if (RetVT.SimpleTy != MVT::nxv2f64)
8627 return Register();
8628 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8629 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8630 }
8631 return Register();
8632}
8633
8634Register fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8635 switch (VT.SimpleTy) {
8636 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
8637 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
8638 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
8639 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
8640 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
8641 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
8642 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
8643 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
8644 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
8645 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8646 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8647 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
8648 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
8649 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
8650 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8651 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8652 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8653 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8654 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8655 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8656 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8657 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8658 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8659 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8660 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8661 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8662 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8663 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8664 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8665 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8666 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8667 default: return Register();
8668 }
8669}
8670
8671// FastEmit functions for AArch64ISD::UZP2.
8672
8673Register fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8674 if (RetVT.SimpleTy != MVT::v8i8)
8675 return Register();
8676 if ((Subtarget->isNeonAvailable())) {
8677 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8678 }
8679 return Register();
8680}
8681
8682Register fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8683 if (RetVT.SimpleTy != MVT::v16i8)
8684 return Register();
8685 if ((Subtarget->isNeonAvailable())) {
8686 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8687 }
8688 return Register();
8689}
8690
8691Register fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
8692 if (RetVT.SimpleTy != MVT::v4i16)
8693 return Register();
8694 if ((Subtarget->isNeonAvailable())) {
8695 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8696 }
8697 return Register();
8698}
8699
8700Register fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8701 if (RetVT.SimpleTy != MVT::v8i16)
8702 return Register();
8703 if ((Subtarget->isNeonAvailable())) {
8704 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8705 }
8706 return Register();
8707}
8708
8709Register fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
8710 if (RetVT.SimpleTy != MVT::v2i32)
8711 return Register();
8712 if ((Subtarget->isNeonAvailable())) {
8713 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8714 }
8715 return Register();
8716}
8717
8718Register fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8719 if (RetVT.SimpleTy != MVT::v4i32)
8720 return Register();
8721 if ((Subtarget->isNeonAvailable())) {
8722 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8723 }
8724 return Register();
8725}
8726
8727Register fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8728 if (RetVT.SimpleTy != MVT::v2i64)
8729 return Register();
8730 if ((Subtarget->isNeonAvailable())) {
8731 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8732 }
8733 return Register();
8734}
8735
8736Register fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8737 if (RetVT.SimpleTy != MVT::v4f16)
8738 return Register();
8739 if ((Subtarget->isNeonAvailable())) {
8740 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8741 }
8742 return Register();
8743}
8744
8745Register fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8746 if (RetVT.SimpleTy != MVT::v8f16)
8747 return Register();
8748 if ((Subtarget->isNeonAvailable())) {
8749 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8750 }
8751 return Register();
8752}
8753
8754Register fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8755 if (RetVT.SimpleTy != MVT::v4bf16)
8756 return Register();
8757 if ((Subtarget->isNeonAvailable())) {
8758 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8759 }
8760 return Register();
8761}
8762
8763Register fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8764 if (RetVT.SimpleTy != MVT::v8bf16)
8765 return Register();
8766 if ((Subtarget->isNeonAvailable())) {
8767 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8768 }
8769 return Register();
8770}
8771
8772Register fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8773 if (RetVT.SimpleTy != MVT::v2f32)
8774 return Register();
8775 if ((Subtarget->isNeonAvailable())) {
8776 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8777 }
8778 return Register();
8779}
8780
8781Register fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8782 if (RetVT.SimpleTy != MVT::v4f32)
8783 return Register();
8784 if ((Subtarget->isNeonAvailable())) {
8785 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8786 }
8787 return Register();
8788}
8789
8790Register fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8791 if (RetVT.SimpleTy != MVT::v2f64)
8792 return Register();
8793 if ((Subtarget->isNeonAvailable())) {
8794 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8795 }
8796 return Register();
8797}
8798
8799Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
8800 if (RetVT.SimpleTy != MVT::nxv2i1)
8801 return Register();
8802 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8803 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8804 }
8805 return Register();
8806}
8807
8808Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
8809 if (RetVT.SimpleTy != MVT::nxv4i1)
8810 return Register();
8811 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8812 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8813 }
8814 return Register();
8815}
8816
8817Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
8818 if (RetVT.SimpleTy != MVT::nxv8i1)
8819 return Register();
8820 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8821 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8822 }
8823 return Register();
8824}
8825
8826Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
8827 if (RetVT.SimpleTy != MVT::nxv16i1)
8828 return Register();
8829 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8830 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8831 }
8832 return Register();
8833}
8834
8835Register fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
8836 if (RetVT.SimpleTy != MVT::nxv16i8)
8837 return Register();
8838 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8839 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8840 }
8841 return Register();
8842}
8843
8844Register fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
8845 if (RetVT.SimpleTy != MVT::nxv8i16)
8846 return Register();
8847 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8848 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8849 }
8850 return Register();
8851}
8852
8853Register fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
8854 if (RetVT.SimpleTy != MVT::nxv4i32)
8855 return Register();
8856 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8857 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8858 }
8859 return Register();
8860}
8861
8862Register fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
8863 if (RetVT.SimpleTy != MVT::nxv2i64)
8864 return Register();
8865 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8866 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8867 }
8868 return Register();
8869}
8870
8871Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
8872 if (RetVT.SimpleTy != MVT::nxv2f16)
8873 return Register();
8874 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8875 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8876 }
8877 return Register();
8878}
8879
8880Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
8881 if (RetVT.SimpleTy != MVT::nxv4f16)
8882 return Register();
8883 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8884 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8885 }
8886 return Register();
8887}
8888
8889Register fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
8890 if (RetVT.SimpleTy != MVT::nxv8f16)
8891 return Register();
8892 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8893 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8894 }
8895 return Register();
8896}
8897
8898Register fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8899 if (RetVT.SimpleTy != MVT::nxv2bf16)
8900 return Register();
8901 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8902 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8903 }
8904 return Register();
8905}
8906
8907Register fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8908 if (RetVT.SimpleTy != MVT::nxv4bf16)
8909 return Register();
8910 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8911 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8912 }
8913 return Register();
8914}
8915
8916Register fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
8917 if (RetVT.SimpleTy != MVT::nxv8bf16)
8918 return Register();
8919 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8920 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8921 }
8922 return Register();
8923}
8924
8925Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
8926 if (RetVT.SimpleTy != MVT::nxv2f32)
8927 return Register();
8928 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8929 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8930 }
8931 return Register();
8932}
8933
8934Register fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
8935 if (RetVT.SimpleTy != MVT::nxv4f32)
8936 return Register();
8937 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8938 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8939 }
8940 return Register();
8941}
8942
8943Register fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
8944 if (RetVT.SimpleTy != MVT::nxv2f64)
8945 return Register();
8946 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
8947 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8948 }
8949 return Register();
8950}
8951
8952Register fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
8953 switch (VT.SimpleTy) {
8954 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
8955 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
8956 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
8957 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
8958 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
8959 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
8960 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
8961 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
8962 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
8963 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
8964 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
8965 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
8966 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
8967 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
8968 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8969 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8970 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8971 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8972 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8973 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8974 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8975 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8976 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8977 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8978 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8979 case MVT::nxv2bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
8980 case MVT::nxv4bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
8981 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8982 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8983 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8984 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8985 default: return Register();
8986 }
8987}
8988
8989// FastEmit functions for AArch64ISD::ZIP1.
8990
8991Register fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
8992 if (RetVT.SimpleTy != MVT::v8i8)
8993 return Register();
8994 if ((Subtarget->isNeonAvailable())) {
8995 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8996 }
8997 return Register();
8998}
8999
9000Register fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9001 if (RetVT.SimpleTy != MVT::v16i8)
9002 return Register();
9003 if ((Subtarget->isNeonAvailable())) {
9004 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9005 }
9006 return Register();
9007}
9008
9009Register fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9010 if (RetVT.SimpleTy != MVT::v4i16)
9011 return Register();
9012 if ((Subtarget->isNeonAvailable())) {
9013 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9014 }
9015 return Register();
9016}
9017
9018Register fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9019 if (RetVT.SimpleTy != MVT::v8i16)
9020 return Register();
9021 if ((Subtarget->isNeonAvailable())) {
9022 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9023 }
9024 return Register();
9025}
9026
9027Register fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9028 if (RetVT.SimpleTy != MVT::v2i32)
9029 return Register();
9030 if ((Subtarget->isNeonAvailable())) {
9031 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9032 }
9033 return Register();
9034}
9035
9036Register fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9037 if (RetVT.SimpleTy != MVT::v4i32)
9038 return Register();
9039 if ((Subtarget->isNeonAvailable())) {
9040 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9041 }
9042 return Register();
9043}
9044
9045Register fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9046 if (RetVT.SimpleTy != MVT::v2i64)
9047 return Register();
9048 if ((Subtarget->isNeonAvailable())) {
9049 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9050 }
9051 return Register();
9052}
9053
9054Register fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9055 if (RetVT.SimpleTy != MVT::v4f16)
9056 return Register();
9057 if ((Subtarget->isNeonAvailable())) {
9058 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9059 }
9060 return Register();
9061}
9062
9063Register fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9064 if (RetVT.SimpleTy != MVT::v8f16)
9065 return Register();
9066 if ((Subtarget->isNeonAvailable())) {
9067 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9068 }
9069 return Register();
9070}
9071
9072Register fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9073 if (RetVT.SimpleTy != MVT::v4bf16)
9074 return Register();
9075 if ((Subtarget->isNeonAvailable())) {
9076 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9077 }
9078 return Register();
9079}
9080
9081Register fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9082 if (RetVT.SimpleTy != MVT::v8bf16)
9083 return Register();
9084 if ((Subtarget->isNeonAvailable())) {
9085 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9086 }
9087 return Register();
9088}
9089
9090Register fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9091 if (RetVT.SimpleTy != MVT::v2f32)
9092 return Register();
9093 if ((Subtarget->isNeonAvailable())) {
9094 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9095 }
9096 return Register();
9097}
9098
9099Register fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9100 if (RetVT.SimpleTy != MVT::v4f32)
9101 return Register();
9102 if ((Subtarget->isNeonAvailable())) {
9103 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9104 }
9105 return Register();
9106}
9107
9108Register fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9109 if (RetVT.SimpleTy != MVT::v2f64)
9110 return Register();
9111 if ((Subtarget->isNeonAvailable())) {
9112 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9113 }
9114 return Register();
9115}
9116
9117Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9118 if (RetVT.SimpleTy != MVT::nxv2i1)
9119 return Register();
9120 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9121 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9122 }
9123 return Register();
9124}
9125
9126Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9127 if (RetVT.SimpleTy != MVT::nxv4i1)
9128 return Register();
9129 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9130 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9131 }
9132 return Register();
9133}
9134
9135Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9136 if (RetVT.SimpleTy != MVT::nxv8i1)
9137 return Register();
9138 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9139 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9140 }
9141 return Register();
9142}
9143
9144Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9145 if (RetVT.SimpleTy != MVT::nxv16i1)
9146 return Register();
9147 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9148 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9149 }
9150 return Register();
9151}
9152
9153Register fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9154 if (RetVT.SimpleTy != MVT::nxv16i8)
9155 return Register();
9156 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9157 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9158 }
9159 return Register();
9160}
9161
9162Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9163 if (RetVT.SimpleTy != MVT::nxv8i16)
9164 return Register();
9165 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9166 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9167 }
9168 return Register();
9169}
9170
9171Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9172 if (RetVT.SimpleTy != MVT::nxv4i32)
9173 return Register();
9174 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9175 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9176 }
9177 return Register();
9178}
9179
9180Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9181 if (RetVT.SimpleTy != MVT::nxv2i64)
9182 return Register();
9183 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9184 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9185 }
9186 return Register();
9187}
9188
9189Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9190 if (RetVT.SimpleTy != MVT::nxv2f16)
9191 return Register();
9192 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9193 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9194 }
9195 return Register();
9196}
9197
9198Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9199 if (RetVT.SimpleTy != MVT::nxv4f16)
9200 return Register();
9201 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9202 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9203 }
9204 return Register();
9205}
9206
9207Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9208 if (RetVT.SimpleTy != MVT::nxv8f16)
9209 return Register();
9210 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9211 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9212 }
9213 return Register();
9214}
9215
9216Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9217 if (RetVT.SimpleTy != MVT::nxv2bf16)
9218 return Register();
9219 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9220 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9221 }
9222 return Register();
9223}
9224
9225Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9226 if (RetVT.SimpleTy != MVT::nxv4bf16)
9227 return Register();
9228 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9229 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9230 }
9231 return Register();
9232}
9233
9234Register fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9235 if (RetVT.SimpleTy != MVT::nxv8bf16)
9236 return Register();
9237 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9238 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9239 }
9240 return Register();
9241}
9242
9243Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9244 if (RetVT.SimpleTy != MVT::nxv2f32)
9245 return Register();
9246 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9247 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9248 }
9249 return Register();
9250}
9251
9252Register fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9253 if (RetVT.SimpleTy != MVT::nxv4f32)
9254 return Register();
9255 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9256 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9257 }
9258 return Register();
9259}
9260
9261Register fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9262 if (RetVT.SimpleTy != MVT::nxv2f64)
9263 return Register();
9264 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9265 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9266 }
9267 return Register();
9268}
9269
9270Register fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9271 switch (VT.SimpleTy) {
9272 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9273 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9274 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9275 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9276 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9277 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9278 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9279 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9280 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9281 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9282 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9283 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9284 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9285 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9286 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9287 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9288 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9289 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9290 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9291 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9292 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9293 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9294 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9295 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9296 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9297 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9298 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9299 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9300 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9301 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9302 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9303 default: return Register();
9304 }
9305}
9306
9307// FastEmit functions for AArch64ISD::ZIP2.
9308
9309Register fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9310 if (RetVT.SimpleTy != MVT::v8i8)
9311 return Register();
9312 if ((Subtarget->isNeonAvailable())) {
9313 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9314 }
9315 return Register();
9316}
9317
9318Register fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9319 if (RetVT.SimpleTy != MVT::v16i8)
9320 return Register();
9321 if ((Subtarget->isNeonAvailable())) {
9322 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9323 }
9324 return Register();
9325}
9326
9327Register fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9328 if (RetVT.SimpleTy != MVT::v4i16)
9329 return Register();
9330 if ((Subtarget->isNeonAvailable())) {
9331 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9332 }
9333 return Register();
9334}
9335
9336Register fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9337 if (RetVT.SimpleTy != MVT::v8i16)
9338 return Register();
9339 if ((Subtarget->isNeonAvailable())) {
9340 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9341 }
9342 return Register();
9343}
9344
9345Register fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9346 if (RetVT.SimpleTy != MVT::v2i32)
9347 return Register();
9348 if ((Subtarget->isNeonAvailable())) {
9349 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9350 }
9351 return Register();
9352}
9353
9354Register fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9355 if (RetVT.SimpleTy != MVT::v4i32)
9356 return Register();
9357 if ((Subtarget->isNeonAvailable())) {
9358 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9359 }
9360 return Register();
9361}
9362
9363Register fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9364 if (RetVT.SimpleTy != MVT::v2i64)
9365 return Register();
9366 if ((Subtarget->isNeonAvailable())) {
9367 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9368 }
9369 return Register();
9370}
9371
9372Register fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9373 if (RetVT.SimpleTy != MVT::v4f16)
9374 return Register();
9375 if ((Subtarget->isNeonAvailable())) {
9376 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9377 }
9378 return Register();
9379}
9380
9381Register fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9382 if (RetVT.SimpleTy != MVT::v8f16)
9383 return Register();
9384 if ((Subtarget->isNeonAvailable())) {
9385 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9386 }
9387 return Register();
9388}
9389
9390Register fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9391 if (RetVT.SimpleTy != MVT::v4bf16)
9392 return Register();
9393 if ((Subtarget->isNeonAvailable())) {
9394 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9395 }
9396 return Register();
9397}
9398
9399Register fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9400 if (RetVT.SimpleTy != MVT::v8bf16)
9401 return Register();
9402 if ((Subtarget->isNeonAvailable())) {
9403 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9404 }
9405 return Register();
9406}
9407
9408Register fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9409 if (RetVT.SimpleTy != MVT::v2f32)
9410 return Register();
9411 if ((Subtarget->isNeonAvailable())) {
9412 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9413 }
9414 return Register();
9415}
9416
9417Register fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9418 if (RetVT.SimpleTy != MVT::v4f32)
9419 return Register();
9420 if ((Subtarget->isNeonAvailable())) {
9421 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9422 }
9423 return Register();
9424}
9425
9426Register fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9427 if (RetVT.SimpleTy != MVT::v2f64)
9428 return Register();
9429 if ((Subtarget->isNeonAvailable())) {
9430 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9431 }
9432 return Register();
9433}
9434
9435Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
9436 if (RetVT.SimpleTy != MVT::nxv2i1)
9437 return Register();
9438 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9439 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9440 }
9441 return Register();
9442}
9443
9444Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
9445 if (RetVT.SimpleTy != MVT::nxv4i1)
9446 return Register();
9447 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9448 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9449 }
9450 return Register();
9451}
9452
9453Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
9454 if (RetVT.SimpleTy != MVT::nxv8i1)
9455 return Register();
9456 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9457 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9458 }
9459 return Register();
9460}
9461
9462Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, Register Op0, Register Op1) {
9463 if (RetVT.SimpleTy != MVT::nxv16i1)
9464 return Register();
9465 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9466 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9467 }
9468 return Register();
9469}
9470
9471Register fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9472 if (RetVT.SimpleTy != MVT::nxv16i8)
9473 return Register();
9474 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9475 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9476 }
9477 return Register();
9478}
9479
9480Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9481 if (RetVT.SimpleTy != MVT::nxv8i16)
9482 return Register();
9483 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9484 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9485 }
9486 return Register();
9487}
9488
9489Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9490 if (RetVT.SimpleTy != MVT::nxv4i32)
9491 return Register();
9492 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9493 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9494 }
9495 return Register();
9496}
9497
9498Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9499 if (RetVT.SimpleTy != MVT::nxv2i64)
9500 return Register();
9501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9502 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9503 }
9504 return Register();
9505}
9506
9507Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
9508 if (RetVT.SimpleTy != MVT::nxv2f16)
9509 return Register();
9510 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9511 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9512 }
9513 return Register();
9514}
9515
9516Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
9517 if (RetVT.SimpleTy != MVT::nxv4f16)
9518 return Register();
9519 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9520 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9521 }
9522 return Register();
9523}
9524
9525Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
9526 if (RetVT.SimpleTy != MVT::nxv8f16)
9527 return Register();
9528 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9529 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9530 }
9531 return Register();
9532}
9533
9534Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9535 if (RetVT.SimpleTy != MVT::nxv2bf16)
9536 return Register();
9537 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9538 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9539 }
9540 return Register();
9541}
9542
9543Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9544 if (RetVT.SimpleTy != MVT::nxv4bf16)
9545 return Register();
9546 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9547 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9548 }
9549 return Register();
9550}
9551
9552Register fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
9553 if (RetVT.SimpleTy != MVT::nxv8bf16)
9554 return Register();
9555 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9556 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9557 }
9558 return Register();
9559}
9560
9561Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
9562 if (RetVT.SimpleTy != MVT::nxv2f32)
9563 return Register();
9564 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9565 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9566 }
9567 return Register();
9568}
9569
9570Register fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
9571 if (RetVT.SimpleTy != MVT::nxv4f32)
9572 return Register();
9573 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9574 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9575 }
9576 return Register();
9577}
9578
9579Register fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
9580 if (RetVT.SimpleTy != MVT::nxv2f64)
9581 return Register();
9582 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9583 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9584 }
9585 return Register();
9586}
9587
9588Register fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9589 switch (VT.SimpleTy) {
9590 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9591 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9592 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9593 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9594 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9595 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9596 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9597 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9598 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9599 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9600 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9601 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9602 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9603 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9604 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9605 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9606 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9607 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9608 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9609 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9610 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9611 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9612 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9613 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9614 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9615 case MVT::nxv2bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
9616 case MVT::nxv4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
9617 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9618 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9619 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9620 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9621 default: return Register();
9622 }
9623}
9624
9625// FastEmit functions for ISD::ABDS.
9626
9627Register fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9628 if (RetVT.SimpleTy != MVT::v8i8)
9629 return Register();
9630 if ((Subtarget->isNeonAvailable())) {
9631 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9632 }
9633 return Register();
9634}
9635
9636Register fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9637 if (RetVT.SimpleTy != MVT::v16i8)
9638 return Register();
9639 if ((Subtarget->isNeonAvailable())) {
9640 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9641 }
9642 return Register();
9643}
9644
9645Register fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9646 if (RetVT.SimpleTy != MVT::v4i16)
9647 return Register();
9648 if ((Subtarget->isNeonAvailable())) {
9649 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9650 }
9651 return Register();
9652}
9653
9654Register fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9655 if (RetVT.SimpleTy != MVT::v8i16)
9656 return Register();
9657 if ((Subtarget->isNeonAvailable())) {
9658 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9659 }
9660 return Register();
9661}
9662
9663Register fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9664 if (RetVT.SimpleTy != MVT::v2i32)
9665 return Register();
9666 if ((Subtarget->isNeonAvailable())) {
9667 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9668 }
9669 return Register();
9670}
9671
9672Register fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9673 if (RetVT.SimpleTy != MVT::v4i32)
9674 return Register();
9675 if ((Subtarget->isNeonAvailable())) {
9676 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9677 }
9678 return Register();
9679}
9680
9681Register fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9682 switch (VT.SimpleTy) {
9683 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
9684 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
9685 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
9686 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
9687 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
9688 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
9689 default: return Register();
9690 }
9691}
9692
9693// FastEmit functions for ISD::ABDU.
9694
9695Register fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9696 if (RetVT.SimpleTy != MVT::v8i8)
9697 return Register();
9698 if ((Subtarget->isNeonAvailable())) {
9699 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9700 }
9701 return Register();
9702}
9703
9704Register fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9705 if (RetVT.SimpleTy != MVT::v16i8)
9706 return Register();
9707 if ((Subtarget->isNeonAvailable())) {
9708 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9709 }
9710 return Register();
9711}
9712
9713Register fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9714 if (RetVT.SimpleTy != MVT::v4i16)
9715 return Register();
9716 if ((Subtarget->isNeonAvailable())) {
9717 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9718 }
9719 return Register();
9720}
9721
9722Register fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9723 if (RetVT.SimpleTy != MVT::v8i16)
9724 return Register();
9725 if ((Subtarget->isNeonAvailable())) {
9726 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9727 }
9728 return Register();
9729}
9730
9731Register fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9732 if (RetVT.SimpleTy != MVT::v2i32)
9733 return Register();
9734 if ((Subtarget->isNeonAvailable())) {
9735 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9736 }
9737 return Register();
9738}
9739
9740Register fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9741 if (RetVT.SimpleTy != MVT::v4i32)
9742 return Register();
9743 if ((Subtarget->isNeonAvailable())) {
9744 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9745 }
9746 return Register();
9747}
9748
9749Register fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9750 switch (VT.SimpleTy) {
9751 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
9752 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
9753 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
9754 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
9755 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
9756 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
9757 default: return Register();
9758 }
9759}
9760
9761// FastEmit functions for ISD::ADD.
9762
9763Register fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
9764 if (RetVT.SimpleTy != MVT::i32)
9765 return Register();
9766 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
9767}
9768
9769Register fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
9770 if (RetVT.SimpleTy != MVT::i64)
9771 return Register();
9772 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
9773}
9774
9775Register fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9776 if (RetVT.SimpleTy != MVT::v8i8)
9777 return Register();
9778 if ((Subtarget->isNeonAvailable())) {
9779 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9780 }
9781 return Register();
9782}
9783
9784Register fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9785 if (RetVT.SimpleTy != MVT::v16i8)
9786 return Register();
9787 if ((Subtarget->isNeonAvailable())) {
9788 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9789 }
9790 return Register();
9791}
9792
9793Register fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9794 if (RetVT.SimpleTy != MVT::v4i16)
9795 return Register();
9796 if ((Subtarget->isNeonAvailable())) {
9797 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9798 }
9799 return Register();
9800}
9801
9802Register fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9803 if (RetVT.SimpleTy != MVT::v8i16)
9804 return Register();
9805 if ((Subtarget->isNeonAvailable())) {
9806 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9807 }
9808 return Register();
9809}
9810
9811Register fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9812 if (RetVT.SimpleTy != MVT::v2i32)
9813 return Register();
9814 if ((Subtarget->isNeonAvailable())) {
9815 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9816 }
9817 return Register();
9818}
9819
9820Register fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9821 if (RetVT.SimpleTy != MVT::v4i32)
9822 return Register();
9823 if ((Subtarget->isNeonAvailable())) {
9824 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9825 }
9826 return Register();
9827}
9828
9829Register fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
9830 if (RetVT.SimpleTy != MVT::v1i64)
9831 return Register();
9832 if ((Subtarget->isNeonAvailable())) {
9833 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
9834 }
9835 return Register();
9836}
9837
9838Register fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9839 if (RetVT.SimpleTy != MVT::v2i64)
9840 return Register();
9841 if ((Subtarget->isNeonAvailable())) {
9842 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9843 }
9844 return Register();
9845}
9846
9847Register fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9848 if (RetVT.SimpleTy != MVT::nxv16i8)
9849 return Register();
9850 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9851 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9852 }
9853 return Register();
9854}
9855
9856Register fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9857 if (RetVT.SimpleTy != MVT::nxv8i16)
9858 return Register();
9859 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9860 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9861 }
9862 return Register();
9863}
9864
9865Register fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9866 if (RetVT.SimpleTy != MVT::nxv4i32)
9867 return Register();
9868 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9869 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9870 }
9871 return Register();
9872}
9873
9874Register fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9875 if (RetVT.SimpleTy != MVT::nxv2i64)
9876 return Register();
9877 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9878 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9879 }
9880 return Register();
9881}
9882
9883Register fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
9884 switch (VT.SimpleTy) {
9885 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
9886 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
9887 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
9888 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
9889 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
9890 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
9891 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
9892 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
9893 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
9894 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
9895 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9896 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9897 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9898 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9899 default: return Register();
9900 }
9901}
9902
9903// FastEmit functions for ISD::AND.
9904
9905Register fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
9906 if (RetVT.SimpleTy != MVT::i32)
9907 return Register();
9908 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
9909}
9910
9911Register fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
9912 if (RetVT.SimpleTy != MVT::i64)
9913 return Register();
9914 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
9915}
9916
9917Register fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
9918 if (RetVT.SimpleTy != MVT::v8i8)
9919 return Register();
9920 if ((Subtarget->isNeonAvailable())) {
9921 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9922 }
9923 return Register();
9924}
9925
9926Register fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9927 if (RetVT.SimpleTy != MVT::v16i8)
9928 return Register();
9929 if ((Subtarget->isNeonAvailable())) {
9930 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9931 }
9932 return Register();
9933}
9934
9935Register fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
9936 if (RetVT.SimpleTy != MVT::v4i16)
9937 return Register();
9938 if ((Subtarget->isNeonAvailable())) {
9939 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9940 }
9941 return Register();
9942}
9943
9944Register fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9945 if (RetVT.SimpleTy != MVT::v8i16)
9946 return Register();
9947 if ((Subtarget->isNeonAvailable())) {
9948 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9949 }
9950 return Register();
9951}
9952
9953Register fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
9954 if (RetVT.SimpleTy != MVT::v2i32)
9955 return Register();
9956 if ((Subtarget->isNeonAvailable())) {
9957 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9958 }
9959 return Register();
9960}
9961
9962Register fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
9963 if (RetVT.SimpleTy != MVT::v4i32)
9964 return Register();
9965 if ((Subtarget->isNeonAvailable())) {
9966 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9967 }
9968 return Register();
9969}
9970
9971Register fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
9972 if (RetVT.SimpleTy != MVT::v1i64)
9973 return Register();
9974 if ((Subtarget->isNeonAvailable())) {
9975 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9976 }
9977 return Register();
9978}
9979
9980Register fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
9981 if (RetVT.SimpleTy != MVT::v2i64)
9982 return Register();
9983 if ((Subtarget->isNeonAvailable())) {
9984 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9985 }
9986 return Register();
9987}
9988
9989Register fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
9990 if (RetVT.SimpleTy != MVT::nxv16i8)
9991 return Register();
9992 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
9993 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
9994 }
9995 return Register();
9996}
9997
9998Register fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
9999 if (RetVT.SimpleTy != MVT::nxv8i16)
10000 return Register();
10001 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10002 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10003 }
10004 return Register();
10005}
10006
10007Register fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10008 if (RetVT.SimpleTy != MVT::nxv4i32)
10009 return Register();
10010 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10011 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10012 }
10013 return Register();
10014}
10015
10016Register fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
10017 if (RetVT.SimpleTy != MVT::nxv2i64)
10018 return Register();
10019 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10020 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10021 }
10022 return Register();
10023}
10024
10025Register fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10026 switch (VT.SimpleTy) {
10027 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10028 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10029 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10030 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10031 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10032 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10033 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10034 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10035 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10036 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10037 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10038 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10039 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10040 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10041 default: return Register();
10042 }
10043}
10044
10045// FastEmit functions for ISD::AVGCEILS.
10046
10047Register fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10048 if (RetVT.SimpleTy != MVT::v8i8)
10049 return Register();
10050 if ((Subtarget->isNeonAvailable())) {
10051 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10052 }
10053 return Register();
10054}
10055
10056Register fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10057 if (RetVT.SimpleTy != MVT::v16i8)
10058 return Register();
10059 if ((Subtarget->isNeonAvailable())) {
10060 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10061 }
10062 return Register();
10063}
10064
10065Register fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10066 if (RetVT.SimpleTy != MVT::v4i16)
10067 return Register();
10068 if ((Subtarget->isNeonAvailable())) {
10069 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10070 }
10071 return Register();
10072}
10073
10074Register fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10075 if (RetVT.SimpleTy != MVT::v8i16)
10076 return Register();
10077 if ((Subtarget->isNeonAvailable())) {
10078 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10079 }
10080 return Register();
10081}
10082
10083Register fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10084 if (RetVT.SimpleTy != MVT::v2i32)
10085 return Register();
10086 if ((Subtarget->isNeonAvailable())) {
10087 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10088 }
10089 return Register();
10090}
10091
10092Register fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10093 if (RetVT.SimpleTy != MVT::v4i32)
10094 return Register();
10095 if ((Subtarget->isNeonAvailable())) {
10096 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10097 }
10098 return Register();
10099}
10100
10101Register fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10102 switch (VT.SimpleTy) {
10103 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10104 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10105 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10106 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10107 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10108 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10109 default: return Register();
10110 }
10111}
10112
10113// FastEmit functions for ISD::AVGCEILU.
10114
10115Register fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10116 if (RetVT.SimpleTy != MVT::v8i8)
10117 return Register();
10118 if ((Subtarget->isNeonAvailable())) {
10119 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10120 }
10121 return Register();
10122}
10123
10124Register fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10125 if (RetVT.SimpleTy != MVT::v16i8)
10126 return Register();
10127 if ((Subtarget->isNeonAvailable())) {
10128 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10129 }
10130 return Register();
10131}
10132
10133Register fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10134 if (RetVT.SimpleTy != MVT::v4i16)
10135 return Register();
10136 if ((Subtarget->isNeonAvailable())) {
10137 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10138 }
10139 return Register();
10140}
10141
10142Register fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10143 if (RetVT.SimpleTy != MVT::v8i16)
10144 return Register();
10145 if ((Subtarget->isNeonAvailable())) {
10146 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10147 }
10148 return Register();
10149}
10150
10151Register fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10152 if (RetVT.SimpleTy != MVT::v2i32)
10153 return Register();
10154 if ((Subtarget->isNeonAvailable())) {
10155 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10156 }
10157 return Register();
10158}
10159
10160Register fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10161 if (RetVT.SimpleTy != MVT::v4i32)
10162 return Register();
10163 if ((Subtarget->isNeonAvailable())) {
10164 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10165 }
10166 return Register();
10167}
10168
10169Register fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10170 switch (VT.SimpleTy) {
10171 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10172 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10173 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10174 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10175 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10176 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10177 default: return Register();
10178 }
10179}
10180
10181// FastEmit functions for ISD::AVGFLOORS.
10182
10183Register fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10184 if (RetVT.SimpleTy != MVT::v8i8)
10185 return Register();
10186 if ((Subtarget->isNeonAvailable())) {
10187 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10188 }
10189 return Register();
10190}
10191
10192Register fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10193 if (RetVT.SimpleTy != MVT::v16i8)
10194 return Register();
10195 if ((Subtarget->isNeonAvailable())) {
10196 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10197 }
10198 return Register();
10199}
10200
10201Register fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10202 if (RetVT.SimpleTy != MVT::v4i16)
10203 return Register();
10204 if ((Subtarget->isNeonAvailable())) {
10205 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10206 }
10207 return Register();
10208}
10209
10210Register fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10211 if (RetVT.SimpleTy != MVT::v8i16)
10212 return Register();
10213 if ((Subtarget->isNeonAvailable())) {
10214 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10215 }
10216 return Register();
10217}
10218
10219Register fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10220 if (RetVT.SimpleTy != MVT::v2i32)
10221 return Register();
10222 if ((Subtarget->isNeonAvailable())) {
10223 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10224 }
10225 return Register();
10226}
10227
10228Register fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10229 if (RetVT.SimpleTy != MVT::v4i32)
10230 return Register();
10231 if ((Subtarget->isNeonAvailable())) {
10232 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10233 }
10234 return Register();
10235}
10236
10237Register fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10238 switch (VT.SimpleTy) {
10239 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10240 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10241 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10242 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10243 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10244 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10245 default: return Register();
10246 }
10247}
10248
10249// FastEmit functions for ISD::AVGFLOORU.
10250
10251Register fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
10252 if (RetVT.SimpleTy != MVT::v8i8)
10253 return Register();
10254 if ((Subtarget->isNeonAvailable())) {
10255 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10256 }
10257 return Register();
10258}
10259
10260Register fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
10261 if (RetVT.SimpleTy != MVT::v16i8)
10262 return Register();
10263 if ((Subtarget->isNeonAvailable())) {
10264 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10265 }
10266 return Register();
10267}
10268
10269Register fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
10270 if (RetVT.SimpleTy != MVT::v4i16)
10271 return Register();
10272 if ((Subtarget->isNeonAvailable())) {
10273 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10274 }
10275 return Register();
10276}
10277
10278Register fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
10279 if (RetVT.SimpleTy != MVT::v8i16)
10280 return Register();
10281 if ((Subtarget->isNeonAvailable())) {
10282 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10283 }
10284 return Register();
10285}
10286
10287Register fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
10288 if (RetVT.SimpleTy != MVT::v2i32)
10289 return Register();
10290 if ((Subtarget->isNeonAvailable())) {
10291 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10292 }
10293 return Register();
10294}
10295
10296Register fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
10297 if (RetVT.SimpleTy != MVT::v4i32)
10298 return Register();
10299 if ((Subtarget->isNeonAvailable())) {
10300 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10301 }
10302 return Register();
10303}
10304
10305Register fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10306 switch (VT.SimpleTy) {
10307 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10308 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10309 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10310 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10311 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10312 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10313 default: return Register();
10314 }
10315}
10316
10317// FastEmit functions for ISD::CONCAT_VECTORS.
10318
10319Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, Register Op0, Register Op1) {
10320 if (RetVT.SimpleTy != MVT::nxv2i1)
10321 return Register();
10322 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10323 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10324 }
10325 return Register();
10326}
10327
10328Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, Register Op0, Register Op1) {
10329 if (RetVT.SimpleTy != MVT::nxv4i1)
10330 return Register();
10331 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10332 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10333 }
10334 return Register();
10335}
10336
10337Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, Register Op0, Register Op1) {
10338 if (RetVT.SimpleTy != MVT::nxv8i1)
10339 return Register();
10340 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10341 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10342 }
10343 return Register();
10344}
10345
10346Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, Register Op0, Register Op1) {
10347 if (RetVT.SimpleTy != MVT::nxv16i1)
10348 return Register();
10349 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10350 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10351 }
10352 return Register();
10353}
10354
10355Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, Register Op0, Register Op1) {
10356 if (RetVT.SimpleTy != MVT::nxv4f16)
10357 return Register();
10358 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10359 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10360 }
10361 return Register();
10362}
10363
10364Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10365 if (RetVT.SimpleTy != MVT::nxv8f16)
10366 return Register();
10367 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10368 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10369 }
10370 return Register();
10371}
10372
10373Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10374 if (RetVT.SimpleTy != MVT::nxv4bf16)
10375 return Register();
10376 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10377 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10378 }
10379 return Register();
10380}
10381
10382Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10383 if (RetVT.SimpleTy != MVT::nxv8bf16)
10384 return Register();
10385 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10386 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10387 }
10388 return Register();
10389}
10390
10391Register fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10392 if (RetVT.SimpleTy != MVT::nxv4f32)
10393 return Register();
10394 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10395 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10396 }
10397 return Register();
10398}
10399
10400Register fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10401 switch (VT.SimpleTy) {
10402 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
10403 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10404 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10405 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10406 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10407 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10408 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10409 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10410 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10411 default: return Register();
10412 }
10413}
10414
10415// FastEmit functions for ISD::FADD.
10416
10417Register fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10418 if (RetVT.SimpleTy != MVT::f16)
10419 return Register();
10420 if ((Subtarget->hasFullFP16())) {
10421 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10422 }
10423 return Register();
10424}
10425
10426Register fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10427 if (RetVT.SimpleTy != MVT::f32)
10428 return Register();
10429 if ((Subtarget->hasFPARMv8())) {
10430 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10431 }
10432 return Register();
10433}
10434
10435Register fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10436 if (RetVT.SimpleTy != MVT::f64)
10437 return Register();
10438 if ((Subtarget->hasFPARMv8())) {
10439 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10440 }
10441 return Register();
10442}
10443
10444Register fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10445 if (RetVT.SimpleTy != MVT::v4f16)
10446 return Register();
10447 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10448 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10449 }
10450 return Register();
10451}
10452
10453Register fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10454 if (RetVT.SimpleTy != MVT::v8f16)
10455 return Register();
10456 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10457 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10458 }
10459 return Register();
10460}
10461
10462Register fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10463 if (RetVT.SimpleTy != MVT::v2f32)
10464 return Register();
10465 if ((Subtarget->isNeonAvailable())) {
10466 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10467 }
10468 return Register();
10469}
10470
10471Register fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10472 if (RetVT.SimpleTy != MVT::v4f32)
10473 return Register();
10474 if ((Subtarget->isNeonAvailable())) {
10475 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10476 }
10477 return Register();
10478}
10479
10480Register fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10481 if (RetVT.SimpleTy != MVT::v2f64)
10482 return Register();
10483 if ((Subtarget->isNeonAvailable())) {
10484 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10485 }
10486 return Register();
10487}
10488
10489Register fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10490 if (RetVT.SimpleTy != MVT::nxv8f16)
10491 return Register();
10492 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10493 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10494 }
10495 return Register();
10496}
10497
10498Register fastEmit_ISD_FADD_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10499 if (RetVT.SimpleTy != MVT::nxv2bf16)
10500 return Register();
10501 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
10502 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10503 }
10504 return Register();
10505}
10506
10507Register fastEmit_ISD_FADD_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10508 if (RetVT.SimpleTy != MVT::nxv4bf16)
10509 return Register();
10510 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
10511 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10512 }
10513 return Register();
10514}
10515
10516Register fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
10517 if (RetVT.SimpleTy != MVT::nxv8bf16)
10518 return Register();
10519 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
10520 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10521 }
10522 return Register();
10523}
10524
10525Register fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10526 if (RetVT.SimpleTy != MVT::nxv4f32)
10527 return Register();
10528 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10529 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10530 }
10531 return Register();
10532}
10533
10534Register fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10535 if (RetVT.SimpleTy != MVT::nxv2f64)
10536 return Register();
10537 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
10538 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10539 }
10540 return Register();
10541}
10542
10543Register fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10544 switch (VT.SimpleTy) {
10545 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
10546 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
10547 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
10548 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
10549 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
10550 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
10551 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
10552 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
10553 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10554 case MVT::nxv2bf16: return fastEmit_ISD_FADD_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
10555 case MVT::nxv4bf16: return fastEmit_ISD_FADD_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
10556 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10557 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10558 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10559 default: return Register();
10560 }
10561}
10562
10563// FastEmit functions for ISD::FDIV.
10564
10565Register fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10566 if (RetVT.SimpleTy != MVT::f16)
10567 return Register();
10568 if ((Subtarget->hasFullFP16())) {
10569 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10570 }
10571 return Register();
10572}
10573
10574Register fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10575 if (RetVT.SimpleTy != MVT::f32)
10576 return Register();
10577 if ((Subtarget->hasFPARMv8())) {
10578 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10579 }
10580 return Register();
10581}
10582
10583Register fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10584 if (RetVT.SimpleTy != MVT::f64)
10585 return Register();
10586 if ((Subtarget->hasFPARMv8())) {
10587 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10588 }
10589 return Register();
10590}
10591
10592Register fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10593 if (RetVT.SimpleTy != MVT::v4f16)
10594 return Register();
10595 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10596 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10597 }
10598 return Register();
10599}
10600
10601Register fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10602 if (RetVT.SimpleTy != MVT::v8f16)
10603 return Register();
10604 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10605 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10606 }
10607 return Register();
10608}
10609
10610Register fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10611 if (RetVT.SimpleTy != MVT::v2f32)
10612 return Register();
10613 if ((Subtarget->isNeonAvailable())) {
10614 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10615 }
10616 return Register();
10617}
10618
10619Register fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10620 if (RetVT.SimpleTy != MVT::v4f32)
10621 return Register();
10622 if ((Subtarget->isNeonAvailable())) {
10623 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10624 }
10625 return Register();
10626}
10627
10628Register fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10629 if (RetVT.SimpleTy != MVT::v2f64)
10630 return Register();
10631 if ((Subtarget->isNeonAvailable())) {
10632 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10633 }
10634 return Register();
10635}
10636
10637Register fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10638 switch (VT.SimpleTy) {
10639 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
10640 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
10641 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
10642 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
10643 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
10644 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
10645 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
10646 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
10647 default: return Register();
10648 }
10649}
10650
10651// FastEmit functions for ISD::FMAXIMUM.
10652
10653Register fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10654 if (RetVT.SimpleTy != MVT::f16)
10655 return Register();
10656 if ((Subtarget->hasFullFP16())) {
10657 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10658 }
10659 return Register();
10660}
10661
10662Register fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10663 if (RetVT.SimpleTy != MVT::f32)
10664 return Register();
10665 if ((Subtarget->hasFPARMv8())) {
10666 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10667 }
10668 return Register();
10669}
10670
10671Register fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10672 if (RetVT.SimpleTy != MVT::f64)
10673 return Register();
10674 if ((Subtarget->hasFPARMv8())) {
10675 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10676 }
10677 return Register();
10678}
10679
10680Register fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10681 if (RetVT.SimpleTy != MVT::v4f16)
10682 return Register();
10683 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10684 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10685 }
10686 return Register();
10687}
10688
10689Register fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10690 if (RetVT.SimpleTy != MVT::v8f16)
10691 return Register();
10692 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10693 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10694 }
10695 return Register();
10696}
10697
10698Register fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10699 if (RetVT.SimpleTy != MVT::v2f32)
10700 return Register();
10701 if ((Subtarget->isNeonAvailable())) {
10702 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10703 }
10704 return Register();
10705}
10706
10707Register fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10708 if (RetVT.SimpleTy != MVT::v4f32)
10709 return Register();
10710 if ((Subtarget->isNeonAvailable())) {
10711 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10712 }
10713 return Register();
10714}
10715
10716Register fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
10717 if (RetVT.SimpleTy != MVT::v1f64)
10718 return Register();
10719 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10720}
10721
10722Register fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10723 if (RetVT.SimpleTy != MVT::v2f64)
10724 return Register();
10725 if ((Subtarget->isNeonAvailable())) {
10726 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10727 }
10728 return Register();
10729}
10730
10731Register fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10732 switch (VT.SimpleTy) {
10733 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
10734 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
10735 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
10736 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
10737 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
10738 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
10739 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
10740 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
10741 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
10742 default: return Register();
10743 }
10744}
10745
10746// FastEmit functions for ISD::FMAXNUM.
10747
10748Register fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10749 if (RetVT.SimpleTy != MVT::f16)
10750 return Register();
10751 if ((Subtarget->hasFullFP16())) {
10752 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10753 }
10754 return Register();
10755}
10756
10757Register fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10758 if (RetVT.SimpleTy != MVT::f32)
10759 return Register();
10760 if ((Subtarget->hasFPARMv8())) {
10761 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10762 }
10763 return Register();
10764}
10765
10766Register fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10767 if (RetVT.SimpleTy != MVT::f64)
10768 return Register();
10769 if ((Subtarget->hasFPARMv8())) {
10770 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10771 }
10772 return Register();
10773}
10774
10775Register fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10776 if (RetVT.SimpleTy != MVT::v4f16)
10777 return Register();
10778 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10779 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10780 }
10781 return Register();
10782}
10783
10784Register fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10785 if (RetVT.SimpleTy != MVT::v8f16)
10786 return Register();
10787 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10788 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10789 }
10790 return Register();
10791}
10792
10793Register fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10794 if (RetVT.SimpleTy != MVT::v2f32)
10795 return Register();
10796 if ((Subtarget->isNeonAvailable())) {
10797 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10798 }
10799 return Register();
10800}
10801
10802Register fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10803 if (RetVT.SimpleTy != MVT::v4f32)
10804 return Register();
10805 if ((Subtarget->isNeonAvailable())) {
10806 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10807 }
10808 return Register();
10809}
10810
10811Register fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
10812 if (RetVT.SimpleTy != MVT::v1f64)
10813 return Register();
10814 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10815}
10816
10817Register fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10818 if (RetVT.SimpleTy != MVT::v2f64)
10819 return Register();
10820 if ((Subtarget->isNeonAvailable())) {
10821 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10822 }
10823 return Register();
10824}
10825
10826Register fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10827 switch (VT.SimpleTy) {
10828 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
10829 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
10830 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
10831 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
10832 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
10833 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
10834 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
10835 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
10836 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
10837 default: return Register();
10838 }
10839}
10840
10841// FastEmit functions for ISD::FMAXNUM_IEEE.
10842
10843Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10844 if (RetVT.SimpleTy != MVT::f16)
10845 return Register();
10846 if ((Subtarget->hasFullFP16())) {
10847 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10848 }
10849 return Register();
10850}
10851
10852Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10853 if (RetVT.SimpleTy != MVT::f32)
10854 return Register();
10855 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10856}
10857
10858Register fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10859 if (RetVT.SimpleTy != MVT::f64)
10860 return Register();
10861 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10862}
10863
10864Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10865 if (RetVT.SimpleTy != MVT::v4f16)
10866 return Register();
10867 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10868 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10869 }
10870 return Register();
10871}
10872
10873Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10874 if (RetVT.SimpleTy != MVT::v8f16)
10875 return Register();
10876 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10877 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10878 }
10879 return Register();
10880}
10881
10882Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10883 if (RetVT.SimpleTy != MVT::v2f32)
10884 return Register();
10885 if ((Subtarget->isNeonAvailable())) {
10886 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10887 }
10888 return Register();
10889}
10890
10891Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10892 if (RetVT.SimpleTy != MVT::v4f32)
10893 return Register();
10894 if ((Subtarget->isNeonAvailable())) {
10895 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10896 }
10897 return Register();
10898}
10899
10900Register fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10901 if (RetVT.SimpleTy != MVT::v2f64)
10902 return Register();
10903 if ((Subtarget->isNeonAvailable())) {
10904 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10905 }
10906 return Register();
10907}
10908
10909Register fastEmit_ISD_FMAXNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
10910 switch (VT.SimpleTy) {
10911 case MVT::f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
10912 case MVT::f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
10913 case MVT::f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
10914 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
10915 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
10916 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
10917 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
10918 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
10919 default: return Register();
10920 }
10921}
10922
10923// FastEmit functions for ISD::FMINIMUM.
10924
10925Register fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
10926 if (RetVT.SimpleTy != MVT::f16)
10927 return Register();
10928 if ((Subtarget->hasFullFP16())) {
10929 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
10930 }
10931 return Register();
10932}
10933
10934Register fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
10935 if (RetVT.SimpleTy != MVT::f32)
10936 return Register();
10937 if ((Subtarget->hasFPARMv8())) {
10938 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
10939 }
10940 return Register();
10941}
10942
10943Register fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
10944 if (RetVT.SimpleTy != MVT::f64)
10945 return Register();
10946 if ((Subtarget->hasFPARMv8())) {
10947 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10948 }
10949 return Register();
10950}
10951
10952Register fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
10953 if (RetVT.SimpleTy != MVT::v4f16)
10954 return Register();
10955 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10956 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10957 }
10958 return Register();
10959}
10960
10961Register fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
10962 if (RetVT.SimpleTy != MVT::v8f16)
10963 return Register();
10964 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
10965 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10966 }
10967 return Register();
10968}
10969
10970Register fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
10971 if (RetVT.SimpleTy != MVT::v2f32)
10972 return Register();
10973 if ((Subtarget->isNeonAvailable())) {
10974 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10975 }
10976 return Register();
10977}
10978
10979Register fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
10980 if (RetVT.SimpleTy != MVT::v4f32)
10981 return Register();
10982 if ((Subtarget->isNeonAvailable())) {
10983 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10984 }
10985 return Register();
10986}
10987
10988Register fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
10989 if (RetVT.SimpleTy != MVT::v1f64)
10990 return Register();
10991 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
10992}
10993
10994Register fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
10995 if (RetVT.SimpleTy != MVT::v2f64)
10996 return Register();
10997 if ((Subtarget->isNeonAvailable())) {
10998 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10999 }
11000 return Register();
11001}
11002
11003Register fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11004 switch (VT.SimpleTy) {
11005 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11006 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11007 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11008 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11009 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11010 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11011 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11012 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11013 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11014 default: return Register();
11015 }
11016}
11017
11018// FastEmit functions for ISD::FMINNUM.
11019
11020Register fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11021 if (RetVT.SimpleTy != MVT::f16)
11022 return Register();
11023 if ((Subtarget->hasFullFP16())) {
11024 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11025 }
11026 return Register();
11027}
11028
11029Register fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11030 if (RetVT.SimpleTy != MVT::f32)
11031 return Register();
11032 if ((Subtarget->hasFPARMv8())) {
11033 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11034 }
11035 return Register();
11036}
11037
11038Register fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11039 if (RetVT.SimpleTy != MVT::f64)
11040 return Register();
11041 if ((Subtarget->hasFPARMv8())) {
11042 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11043 }
11044 return Register();
11045}
11046
11047Register fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11048 if (RetVT.SimpleTy != MVT::v4f16)
11049 return Register();
11050 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11051 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11052 }
11053 return Register();
11054}
11055
11056Register fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11057 if (RetVT.SimpleTy != MVT::v8f16)
11058 return Register();
11059 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11060 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11061 }
11062 return Register();
11063}
11064
11065Register fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11066 if (RetVT.SimpleTy != MVT::v2f32)
11067 return Register();
11068 if ((Subtarget->isNeonAvailable())) {
11069 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11070 }
11071 return Register();
11072}
11073
11074Register fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11075 if (RetVT.SimpleTy != MVT::v4f32)
11076 return Register();
11077 if ((Subtarget->isNeonAvailable())) {
11078 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11079 }
11080 return Register();
11081}
11082
11083Register fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, Register Op0, Register Op1) {
11084 if (RetVT.SimpleTy != MVT::v1f64)
11085 return Register();
11086 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11087}
11088
11089Register fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11090 if (RetVT.SimpleTy != MVT::v2f64)
11091 return Register();
11092 if ((Subtarget->isNeonAvailable())) {
11093 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11094 }
11095 return Register();
11096}
11097
11098Register fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11099 switch (VT.SimpleTy) {
11100 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11101 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11102 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11103 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11104 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11105 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11106 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11107 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11108 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11109 default: return Register();
11110 }
11111}
11112
11113// FastEmit functions for ISD::FMINNUM_IEEE.
11114
11115Register fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11116 if (RetVT.SimpleTy != MVT::f16)
11117 return Register();
11118 if ((Subtarget->hasFullFP16())) {
11119 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11120 }
11121 return Register();
11122}
11123
11124Register fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11125 if (RetVT.SimpleTy != MVT::f32)
11126 return Register();
11127 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11128}
11129
11130Register fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11131 if (RetVT.SimpleTy != MVT::f64)
11132 return Register();
11133 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11134}
11135
11136Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11137 if (RetVT.SimpleTy != MVT::v4f16)
11138 return Register();
11139 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11140 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11141 }
11142 return Register();
11143}
11144
11145Register fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11146 if (RetVT.SimpleTy != MVT::v8f16)
11147 return Register();
11148 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11149 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11150 }
11151 return Register();
11152}
11153
11154Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11155 if (RetVT.SimpleTy != MVT::v2f32)
11156 return Register();
11157 if ((Subtarget->isNeonAvailable())) {
11158 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11159 }
11160 return Register();
11161}
11162
11163Register fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11164 if (RetVT.SimpleTy != MVT::v4f32)
11165 return Register();
11166 if ((Subtarget->isNeonAvailable())) {
11167 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11168 }
11169 return Register();
11170}
11171
11172Register fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11173 if (RetVT.SimpleTy != MVT::v2f64)
11174 return Register();
11175 if ((Subtarget->isNeonAvailable())) {
11176 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11177 }
11178 return Register();
11179}
11180
11181Register fastEmit_ISD_FMINNUM_IEEE_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11182 switch (VT.SimpleTy) {
11183 case MVT::f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_f16_rr(RetVT, Op0, Op1);
11184 case MVT::f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_f32_rr(RetVT, Op0, Op1);
11185 case MVT::f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_f64_rr(RetVT, Op0, Op1);
11186 case MVT::v4f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f16_rr(RetVT, Op0, Op1);
11187 case MVT::v8f16: return fastEmit_ISD_FMINNUM_IEEE_MVT_v8f16_rr(RetVT, Op0, Op1);
11188 case MVT::v2f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f32_rr(RetVT, Op0, Op1);
11189 case MVT::v4f32: return fastEmit_ISD_FMINNUM_IEEE_MVT_v4f32_rr(RetVT, Op0, Op1);
11190 case MVT::v2f64: return fastEmit_ISD_FMINNUM_IEEE_MVT_v2f64_rr(RetVT, Op0, Op1);
11191 default: return Register();
11192 }
11193}
11194
11195// FastEmit functions for ISD::FMUL.
11196
11197Register fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11198 if (RetVT.SimpleTy != MVT::f16)
11199 return Register();
11200 if ((Subtarget->hasFullFP16())) {
11201 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11202 }
11203 return Register();
11204}
11205
11206Register fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11207 if (RetVT.SimpleTy != MVT::f32)
11208 return Register();
11209 if ((Subtarget->hasFPARMv8())) {
11210 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11211 }
11212 return Register();
11213}
11214
11215Register fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11216 if (RetVT.SimpleTy != MVT::f64)
11217 return Register();
11218 if ((Subtarget->hasFPARMv8())) {
11219 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11220 }
11221 return Register();
11222}
11223
11224Register fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11225 if (RetVT.SimpleTy != MVT::v4f16)
11226 return Register();
11227 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11228 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11229 }
11230 return Register();
11231}
11232
11233Register fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11234 if (RetVT.SimpleTy != MVT::v8f16)
11235 return Register();
11236 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11237 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11238 }
11239 return Register();
11240}
11241
11242Register fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11243 if (RetVT.SimpleTy != MVT::v2f32)
11244 return Register();
11245 if ((Subtarget->isNeonAvailable())) {
11246 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11247 }
11248 return Register();
11249}
11250
11251Register fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11252 if (RetVT.SimpleTy != MVT::v4f32)
11253 return Register();
11254 if ((Subtarget->isNeonAvailable())) {
11255 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11256 }
11257 return Register();
11258}
11259
11260Register fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11261 if (RetVT.SimpleTy != MVT::v2f64)
11262 return Register();
11263 if ((Subtarget->isNeonAvailable())) {
11264 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11265 }
11266 return Register();
11267}
11268
11269Register fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11270 if (RetVT.SimpleTy != MVT::nxv8f16)
11271 return Register();
11272 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11273 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11274 }
11275 return Register();
11276}
11277
11278Register fastEmit_ISD_FMUL_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11279 if (RetVT.SimpleTy != MVT::nxv2bf16)
11280 return Register();
11281 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
11282 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11283 }
11284 return Register();
11285}
11286
11287Register fastEmit_ISD_FMUL_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11288 if (RetVT.SimpleTy != MVT::nxv4bf16)
11289 return Register();
11290 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
11291 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11292 }
11293 return Register();
11294}
11295
11296Register fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11297 if (RetVT.SimpleTy != MVT::nxv8bf16)
11298 return Register();
11299 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
11300 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11301 }
11302 return Register();
11303}
11304
11305Register fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11306 if (RetVT.SimpleTy != MVT::nxv4f32)
11307 return Register();
11308 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11309 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11310 }
11311 return Register();
11312}
11313
11314Register fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11315 if (RetVT.SimpleTy != MVT::nxv2f64)
11316 return Register();
11317 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11318 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11319 }
11320 return Register();
11321}
11322
11323Register fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11324 switch (VT.SimpleTy) {
11325 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11326 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11327 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11328 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11329 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11330 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11331 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11332 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11333 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11334 case MVT::nxv2bf16: return fastEmit_ISD_FMUL_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
11335 case MVT::nxv4bf16: return fastEmit_ISD_FMUL_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
11336 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11337 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11338 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11339 default: return Register();
11340 }
11341}
11342
11343// FastEmit functions for ISD::FSUB.
11344
11345Register fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
11346 if (RetVT.SimpleTy != MVT::f16)
11347 return Register();
11348 if ((Subtarget->hasFullFP16())) {
11349 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11350 }
11351 return Register();
11352}
11353
11354Register fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
11355 if (RetVT.SimpleTy != MVT::f32)
11356 return Register();
11357 if ((Subtarget->hasFPARMv8())) {
11358 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11359 }
11360 return Register();
11361}
11362
11363Register fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
11364 if (RetVT.SimpleTy != MVT::f64)
11365 return Register();
11366 if ((Subtarget->hasFPARMv8())) {
11367 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11368 }
11369 return Register();
11370}
11371
11372Register fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
11373 if (RetVT.SimpleTy != MVT::v4f16)
11374 return Register();
11375 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11376 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11377 }
11378 return Register();
11379}
11380
11381Register fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11382 if (RetVT.SimpleTy != MVT::v8f16)
11383 return Register();
11384 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11385 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11386 }
11387 return Register();
11388}
11389
11390Register fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
11391 if (RetVT.SimpleTy != MVT::v2f32)
11392 return Register();
11393 if ((Subtarget->isNeonAvailable())) {
11394 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11395 }
11396 return Register();
11397}
11398
11399Register fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11400 if (RetVT.SimpleTy != MVT::v4f32)
11401 return Register();
11402 if ((Subtarget->isNeonAvailable())) {
11403 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11404 }
11405 return Register();
11406}
11407
11408Register fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11409 if (RetVT.SimpleTy != MVT::v2f64)
11410 return Register();
11411 if ((Subtarget->isNeonAvailable())) {
11412 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11413 }
11414 return Register();
11415}
11416
11417Register fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, Register Op0, Register Op1) {
11418 if (RetVT.SimpleTy != MVT::nxv8f16)
11419 return Register();
11420 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11421 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11422 }
11423 return Register();
11424}
11425
11426Register fastEmit_ISD_FSUB_MVT_nxv2bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11427 if (RetVT.SimpleTy != MVT::nxv2bf16)
11428 return Register();
11429 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
11430 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11431 }
11432 return Register();
11433}
11434
11435Register fastEmit_ISD_FSUB_MVT_nxv4bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11436 if (RetVT.SimpleTy != MVT::nxv4bf16)
11437 return Register();
11438 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
11439 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11440 }
11441 return Register();
11442}
11443
11444Register fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, Register Op0, Register Op1) {
11445 if (RetVT.SimpleTy != MVT::nxv8bf16)
11446 return Register();
11447 if ((Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16())) {
11448 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11449 }
11450 return Register();
11451}
11452
11453Register fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, Register Op0, Register Op1) {
11454 if (RetVT.SimpleTy != MVT::nxv4f32)
11455 return Register();
11456 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11457 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11458 }
11459 return Register();
11460}
11461
11462Register fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, Register Op0, Register Op1) {
11463 if (RetVT.SimpleTy != MVT::nxv2f64)
11464 return Register();
11465 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11466 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11467 }
11468 return Register();
11469}
11470
11471Register fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11472 switch (VT.SimpleTy) {
11473 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11474 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11475 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11476 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11477 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11478 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11479 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11480 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11481 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11482 case MVT::nxv2bf16: return fastEmit_ISD_FSUB_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
11483 case MVT::nxv4bf16: return fastEmit_ISD_FSUB_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
11484 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11485 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11486 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11487 default: return Register();
11488 }
11489}
11490
11491// FastEmit functions for ISD::GET_ACTIVE_LANE_MASK.
11492
11493Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11494 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11495 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11496 }
11497 return Register();
11498}
11499
11500Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11501 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11502 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11503 }
11504 return Register();
11505}
11506
11507Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11508 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11509 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11510 }
11511 return Register();
11512}
11513
11514Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11515 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11516 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PWW_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11517 }
11518 return Register();
11519}
11520
11521Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11522switch (RetVT.SimpleTy) {
11523 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv2i1_rr(Op0, Op1);
11524 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv4i1_rr(Op0, Op1);
11525 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv8i1_rr(Op0, Op1);
11526 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_MVT_nxv16i1_rr(Op0, Op1);
11527 default: return Register();
11528}
11529}
11530
11531Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Register Op0, Register Op1) {
11532 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11533 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_D, RC: &AArch64::PPRRegClass, Op0, Op1);
11534 }
11535 return Register();
11536}
11537
11538Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Register Op0, Register Op1) {
11539 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11540 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_S, RC: &AArch64::PPRRegClass, Op0, Op1);
11541 }
11542 return Register();
11543}
11544
11545Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Register Op0, Register Op1) {
11546 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11547 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_H, RC: &AArch64::PPRRegClass, Op0, Op1);
11548 }
11549 return Register();
11550}
11551
11552Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Register Op0, Register Op1) {
11553 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11554 return fastEmitInst_rr(MachineInstOpcode: AArch64::WHILELO_PXX_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11555 }
11556 return Register();
11557}
11558
11559Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11560switch (RetVT.SimpleTy) {
11561 case MVT::nxv2i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv2i1_rr(Op0, Op1);
11562 case MVT::nxv4i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv4i1_rr(Op0, Op1);
11563 case MVT::nxv8i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv8i1_rr(Op0, Op1);
11564 case MVT::nxv16i1: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_MVT_nxv16i1_rr(Op0, Op1);
11565 default: return Register();
11566}
11567}
11568
11569Register fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11570 switch (VT.SimpleTy) {
11571 case MVT::i32: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i32_rr(RetVT, Op0, Op1);
11572 case MVT::i64: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_MVT_i64_rr(RetVT, Op0, Op1);
11573 default: return Register();
11574 }
11575}
11576
11577// FastEmit functions for ISD::MUL.
11578
11579Register fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11580 if (RetVT.SimpleTy != MVT::v8i8)
11581 return Register();
11582 if ((Subtarget->isNeonAvailable())) {
11583 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11584 }
11585 return Register();
11586}
11587
11588Register fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11589 if (RetVT.SimpleTy != MVT::v16i8)
11590 return Register();
11591 if ((Subtarget->isNeonAvailable())) {
11592 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11593 }
11594 return Register();
11595}
11596
11597Register fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11598 if (RetVT.SimpleTy != MVT::v4i16)
11599 return Register();
11600 if ((Subtarget->isNeonAvailable())) {
11601 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11602 }
11603 return Register();
11604}
11605
11606Register fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11607 if (RetVT.SimpleTy != MVT::v8i16)
11608 return Register();
11609 if ((Subtarget->isNeonAvailable())) {
11610 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11611 }
11612 return Register();
11613}
11614
11615Register fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11616 if (RetVT.SimpleTy != MVT::v2i32)
11617 return Register();
11618 if ((Subtarget->isNeonAvailable())) {
11619 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11620 }
11621 return Register();
11622}
11623
11624Register fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11625 if (RetVT.SimpleTy != MVT::v4i32)
11626 return Register();
11627 if ((Subtarget->isNeonAvailable())) {
11628 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11629 }
11630 return Register();
11631}
11632
11633Register fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11634 switch (VT.SimpleTy) {
11635 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
11636 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
11637 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
11638 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11639 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
11640 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11641 default: return Register();
11642 }
11643}
11644
11645// FastEmit functions for ISD::MULHS.
11646
11647Register fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11648 if (RetVT.SimpleTy != MVT::i64)
11649 return Register();
11650 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11651}
11652
11653Register fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11654 switch (VT.SimpleTy) {
11655 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
11656 default: return Register();
11657 }
11658}
11659
11660// FastEmit functions for ISD::MULHU.
11661
11662Register fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11663 if (RetVT.SimpleTy != MVT::i64)
11664 return Register();
11665 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11666}
11667
11668Register fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11669 switch (VT.SimpleTy) {
11670 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
11671 default: return Register();
11672 }
11673}
11674
11675// FastEmit functions for ISD::OR.
11676
11677Register fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11678 if (RetVT.SimpleTy != MVT::i32)
11679 return Register();
11680 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
11681}
11682
11683Register fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11684 if (RetVT.SimpleTy != MVT::i64)
11685 return Register();
11686 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11687}
11688
11689Register fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11690 if (RetVT.SimpleTy != MVT::v8i8)
11691 return Register();
11692 if ((Subtarget->isNeonAvailable())) {
11693 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11694 }
11695 return Register();
11696}
11697
11698Register fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11699 if (RetVT.SimpleTy != MVT::v16i8)
11700 return Register();
11701 if ((Subtarget->isNeonAvailable())) {
11702 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11703 }
11704 return Register();
11705}
11706
11707Register fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11708 if (RetVT.SimpleTy != MVT::v4i16)
11709 return Register();
11710 if ((Subtarget->isNeonAvailable())) {
11711 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11712 }
11713 return Register();
11714}
11715
11716Register fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11717 if (RetVT.SimpleTy != MVT::v8i16)
11718 return Register();
11719 if ((Subtarget->isNeonAvailable())) {
11720 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11721 }
11722 return Register();
11723}
11724
11725Register fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11726 if (RetVT.SimpleTy != MVT::v2i32)
11727 return Register();
11728 if ((Subtarget->isNeonAvailable())) {
11729 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11730 }
11731 return Register();
11732}
11733
11734Register fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11735 if (RetVT.SimpleTy != MVT::v4i32)
11736 return Register();
11737 if ((Subtarget->isNeonAvailable())) {
11738 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11739 }
11740 return Register();
11741}
11742
11743Register fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
11744 if (RetVT.SimpleTy != MVT::v1i64)
11745 return Register();
11746 if ((Subtarget->isNeonAvailable())) {
11747 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11748 }
11749 return Register();
11750}
11751
11752Register fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11753 if (RetVT.SimpleTy != MVT::v2i64)
11754 return Register();
11755 if ((Subtarget->isNeonAvailable())) {
11756 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11757 }
11758 return Register();
11759}
11760
11761Register fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11762 if (RetVT.SimpleTy != MVT::nxv16i8)
11763 return Register();
11764 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11765 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11766 }
11767 return Register();
11768}
11769
11770Register fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11771 if (RetVT.SimpleTy != MVT::nxv8i16)
11772 return Register();
11773 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11774 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11775 }
11776 return Register();
11777}
11778
11779Register fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11780 if (RetVT.SimpleTy != MVT::nxv4i32)
11781 return Register();
11782 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11783 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11784 }
11785 return Register();
11786}
11787
11788Register fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11789 if (RetVT.SimpleTy != MVT::nxv2i64)
11790 return Register();
11791 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11792 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11793 }
11794 return Register();
11795}
11796
11797Register fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11798 switch (VT.SimpleTy) {
11799 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
11800 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
11801 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
11802 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
11803 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
11804 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
11805 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
11806 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
11807 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
11808 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
11809 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
11810 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
11811 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
11812 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
11813 default: return Register();
11814 }
11815}
11816
11817// FastEmit functions for ISD::ROTR.
11818
11819Register fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11820 if (RetVT.SimpleTy != MVT::i64)
11821 return Register();
11822 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11823}
11824
11825Register fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11826 switch (VT.SimpleTy) {
11827 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
11828 default: return Register();
11829 }
11830}
11831
11832// FastEmit functions for ISD::SADDSAT.
11833
11834Register fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
11835 if (RetVT.SimpleTy != MVT::v8i8)
11836 return Register();
11837 if ((Subtarget->isNeonAvailable())) {
11838 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11839 }
11840 return Register();
11841}
11842
11843Register fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11844 if (RetVT.SimpleTy != MVT::v16i8)
11845 return Register();
11846 if ((Subtarget->isNeonAvailable())) {
11847 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11848 }
11849 return Register();
11850}
11851
11852Register fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
11853 if (RetVT.SimpleTy != MVT::v4i16)
11854 return Register();
11855 if ((Subtarget->isNeonAvailable())) {
11856 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11857 }
11858 return Register();
11859}
11860
11861Register fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11862 if (RetVT.SimpleTy != MVT::v8i16)
11863 return Register();
11864 if ((Subtarget->isNeonAvailable())) {
11865 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11866 }
11867 return Register();
11868}
11869
11870Register fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
11871 if (RetVT.SimpleTy != MVT::v2i32)
11872 return Register();
11873 if ((Subtarget->isNeonAvailable())) {
11874 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11875 }
11876 return Register();
11877}
11878
11879Register fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11880 if (RetVT.SimpleTy != MVT::v4i32)
11881 return Register();
11882 if ((Subtarget->isNeonAvailable())) {
11883 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11884 }
11885 return Register();
11886}
11887
11888Register fastEmit_ISD_SADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
11889 if (RetVT.SimpleTy != MVT::v1i64)
11890 return Register();
11891 if ((Subtarget->isNeonAvailable())) {
11892 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
11893 }
11894 return Register();
11895}
11896
11897Register fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11898 if (RetVT.SimpleTy != MVT::v2i64)
11899 return Register();
11900 if ((Subtarget->isNeonAvailable())) {
11901 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11902 }
11903 return Register();
11904}
11905
11906Register fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
11907 if (RetVT.SimpleTy != MVT::nxv16i8)
11908 return Register();
11909 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11910 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
11911 }
11912 return Register();
11913}
11914
11915Register fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
11916 if (RetVT.SimpleTy != MVT::nxv8i16)
11917 return Register();
11918 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11919 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11920 }
11921 return Register();
11922}
11923
11924Register fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
11925 if (RetVT.SimpleTy != MVT::nxv4i32)
11926 return Register();
11927 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11928 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11929 }
11930 return Register();
11931}
11932
11933Register fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
11934 if (RetVT.SimpleTy != MVT::nxv2i64)
11935 return Register();
11936 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
11937 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11938 }
11939 return Register();
11940}
11941
11942Register fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11943 switch (VT.SimpleTy) {
11944 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
11945 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
11946 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
11947 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
11948 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
11949 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
11950 case MVT::v1i64: return fastEmit_ISD_SADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
11951 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
11952 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
11953 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
11954 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
11955 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
11956 default: return Register();
11957 }
11958}
11959
11960// FastEmit functions for ISD::SDIV.
11961
11962Register fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
11963 if (RetVT.SimpleTy != MVT::i32)
11964 return Register();
11965 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
11966}
11967
11968Register fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11969 if (RetVT.SimpleTy != MVT::i64)
11970 return Register();
11971 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11972}
11973
11974Register fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11975 switch (VT.SimpleTy) {
11976 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
11977 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
11978 default: return Register();
11979 }
11980}
11981
11982// FastEmit functions for ISD::SHL.
11983
11984Register fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
11985 if (RetVT.SimpleTy != MVT::i64)
11986 return Register();
11987 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11988}
11989
11990Register fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
11991 switch (VT.SimpleTy) {
11992 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
11993 default: return Register();
11994 }
11995}
11996
11997// FastEmit functions for ISD::SMAX.
11998
11999Register fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12000 if (RetVT.SimpleTy != MVT::i32)
12001 return Register();
12002 if ((Subtarget->hasCSSC())) {
12003 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12004 }
12005 return Register();
12006}
12007
12008Register fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12009 if (RetVT.SimpleTy != MVT::i64)
12010 return Register();
12011 if ((Subtarget->hasCSSC())) {
12012 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12013 }
12014 return Register();
12015}
12016
12017Register fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12018 if (RetVT.SimpleTy != MVT::v8i8)
12019 return Register();
12020 if ((Subtarget->isNeonAvailable())) {
12021 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12022 }
12023 return Register();
12024}
12025
12026Register fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12027 if (RetVT.SimpleTy != MVT::v16i8)
12028 return Register();
12029 if ((Subtarget->isNeonAvailable())) {
12030 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12031 }
12032 return Register();
12033}
12034
12035Register fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12036 if (RetVT.SimpleTy != MVT::v4i16)
12037 return Register();
12038 if ((Subtarget->isNeonAvailable())) {
12039 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12040 }
12041 return Register();
12042}
12043
12044Register fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12045 if (RetVT.SimpleTy != MVT::v8i16)
12046 return Register();
12047 if ((Subtarget->isNeonAvailable())) {
12048 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12049 }
12050 return Register();
12051}
12052
12053Register fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12054 if (RetVT.SimpleTy != MVT::v2i32)
12055 return Register();
12056 if ((Subtarget->isNeonAvailable())) {
12057 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12058 }
12059 return Register();
12060}
12061
12062Register fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12063 if (RetVT.SimpleTy != MVT::v4i32)
12064 return Register();
12065 if ((Subtarget->isNeonAvailable())) {
12066 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12067 }
12068 return Register();
12069}
12070
12071Register fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12072 switch (VT.SimpleTy) {
12073 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12074 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12075 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12076 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12077 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12078 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12079 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12080 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12081 default: return Register();
12082 }
12083}
12084
12085// FastEmit functions for ISD::SMIN.
12086
12087Register fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
12088 if (RetVT.SimpleTy != MVT::i32)
12089 return Register();
12090 if ((Subtarget->hasCSSC())) {
12091 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12092 }
12093 return Register();
12094}
12095
12096Register fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12097 if (RetVT.SimpleTy != MVT::i64)
12098 return Register();
12099 if ((Subtarget->hasCSSC())) {
12100 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12101 }
12102 return Register();
12103}
12104
12105Register fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12106 if (RetVT.SimpleTy != MVT::v8i8)
12107 return Register();
12108 if ((Subtarget->isNeonAvailable())) {
12109 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12110 }
12111 return Register();
12112}
12113
12114Register fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12115 if (RetVT.SimpleTy != MVT::v16i8)
12116 return Register();
12117 if ((Subtarget->isNeonAvailable())) {
12118 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12119 }
12120 return Register();
12121}
12122
12123Register fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12124 if (RetVT.SimpleTy != MVT::v4i16)
12125 return Register();
12126 if ((Subtarget->isNeonAvailable())) {
12127 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12128 }
12129 return Register();
12130}
12131
12132Register fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12133 if (RetVT.SimpleTy != MVT::v8i16)
12134 return Register();
12135 if ((Subtarget->isNeonAvailable())) {
12136 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12137 }
12138 return Register();
12139}
12140
12141Register fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12142 if (RetVT.SimpleTy != MVT::v2i32)
12143 return Register();
12144 if ((Subtarget->isNeonAvailable())) {
12145 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12146 }
12147 return Register();
12148}
12149
12150Register fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12151 if (RetVT.SimpleTy != MVT::v4i32)
12152 return Register();
12153 if ((Subtarget->isNeonAvailable())) {
12154 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12155 }
12156 return Register();
12157}
12158
12159Register fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12160 switch (VT.SimpleTy) {
12161 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12162 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12163 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12164 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12165 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12166 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12167 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12168 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12169 default: return Register();
12170 }
12171}
12172
12173// FastEmit functions for ISD::SRA.
12174
12175Register fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12176 if (RetVT.SimpleTy != MVT::i64)
12177 return Register();
12178 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12179}
12180
12181Register fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12182 switch (VT.SimpleTy) {
12183 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12184 default: return Register();
12185 }
12186}
12187
12188// FastEmit functions for ISD::SRL.
12189
12190Register fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
12191 if (RetVT.SimpleTy != MVT::i64)
12192 return Register();
12193 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12194}
12195
12196Register fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12197 switch (VT.SimpleTy) {
12198 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12199 default: return Register();
12200 }
12201}
12202
12203// FastEmit functions for ISD::SSUBSAT.
12204
12205Register fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
12206 if (RetVT.SimpleTy != MVT::v8i8)
12207 return Register();
12208 if ((Subtarget->isNeonAvailable())) {
12209 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12210 }
12211 return Register();
12212}
12213
12214Register fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12215 if (RetVT.SimpleTy != MVT::v16i8)
12216 return Register();
12217 if ((Subtarget->isNeonAvailable())) {
12218 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12219 }
12220 return Register();
12221}
12222
12223Register fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
12224 if (RetVT.SimpleTy != MVT::v4i16)
12225 return Register();
12226 if ((Subtarget->isNeonAvailable())) {
12227 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12228 }
12229 return Register();
12230}
12231
12232Register fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12233 if (RetVT.SimpleTy != MVT::v8i16)
12234 return Register();
12235 if ((Subtarget->isNeonAvailable())) {
12236 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12237 }
12238 return Register();
12239}
12240
12241Register fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
12242 if (RetVT.SimpleTy != MVT::v2i32)
12243 return Register();
12244 if ((Subtarget->isNeonAvailable())) {
12245 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12246 }
12247 return Register();
12248}
12249
12250Register fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12251 if (RetVT.SimpleTy != MVT::v4i32)
12252 return Register();
12253 if ((Subtarget->isNeonAvailable())) {
12254 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12255 }
12256 return Register();
12257}
12258
12259Register fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
12260 if (RetVT.SimpleTy != MVT::v1i64)
12261 return Register();
12262 if ((Subtarget->isNeonAvailable())) {
12263 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
12264 }
12265 return Register();
12266}
12267
12268Register fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12269 if (RetVT.SimpleTy != MVT::v2i64)
12270 return Register();
12271 if ((Subtarget->isNeonAvailable())) {
12272 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12273 }
12274 return Register();
12275}
12276
12277Register fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
12278 if (RetVT.SimpleTy != MVT::nxv16i8)
12279 return Register();
12280 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12281 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12282 }
12283 return Register();
12284}
12285
12286Register fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
12287 if (RetVT.SimpleTy != MVT::nxv8i16)
12288 return Register();
12289 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12290 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12291 }
12292 return Register();
12293}
12294
12295Register fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
12296 if (RetVT.SimpleTy != MVT::nxv4i32)
12297 return Register();
12298 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12299 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12300 }
12301 return Register();
12302}
12303
12304Register fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
12305 if (RetVT.SimpleTy != MVT::nxv2i64)
12306 return Register();
12307 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
12308 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12309 }
12310 return Register();
12311}
12312
12313Register fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12314 switch (VT.SimpleTy) {
12315 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12316 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12317 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12318 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12319 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12320 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12321 case MVT::v1i64: return fastEmit_ISD_SSUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
12322 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12323 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12324 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12325 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12326 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12327 default: return Register();
12328 }
12329}
12330
12331// FastEmit functions for ISD::STRICT_FADD.
12332
12333Register fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12334 if (RetVT.SimpleTy != MVT::f16)
12335 return Register();
12336 if ((Subtarget->hasFullFP16())) {
12337 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12338 }
12339 return Register();
12340}
12341
12342Register fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12343 if (RetVT.SimpleTy != MVT::f32)
12344 return Register();
12345 if ((Subtarget->hasFPARMv8())) {
12346 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12347 }
12348 return Register();
12349}
12350
12351Register fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12352 if (RetVT.SimpleTy != MVT::f64)
12353 return Register();
12354 if ((Subtarget->hasFPARMv8())) {
12355 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12356 }
12357 return Register();
12358}
12359
12360Register fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12361 if (RetVT.SimpleTy != MVT::v4f16)
12362 return Register();
12363 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12364 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12365 }
12366 return Register();
12367}
12368
12369Register fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12370 if (RetVT.SimpleTy != MVT::v8f16)
12371 return Register();
12372 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12373 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12374 }
12375 return Register();
12376}
12377
12378Register fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12379 if (RetVT.SimpleTy != MVT::v2f32)
12380 return Register();
12381 if ((Subtarget->isNeonAvailable())) {
12382 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12383 }
12384 return Register();
12385}
12386
12387Register fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12388 if (RetVT.SimpleTy != MVT::v4f32)
12389 return Register();
12390 if ((Subtarget->isNeonAvailable())) {
12391 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12392 }
12393 return Register();
12394}
12395
12396Register fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12397 if (RetVT.SimpleTy != MVT::v2f64)
12398 return Register();
12399 if ((Subtarget->isNeonAvailable())) {
12400 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12401 }
12402 return Register();
12403}
12404
12405Register fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12406 switch (VT.SimpleTy) {
12407 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12408 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12409 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12410 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12411 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12412 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12413 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12414 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12415 default: return Register();
12416 }
12417}
12418
12419// FastEmit functions for ISD::STRICT_FDIV.
12420
12421Register fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12422 if (RetVT.SimpleTy != MVT::f16)
12423 return Register();
12424 if ((Subtarget->hasFullFP16())) {
12425 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12426 }
12427 return Register();
12428}
12429
12430Register fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12431 if (RetVT.SimpleTy != MVT::f32)
12432 return Register();
12433 if ((Subtarget->hasFPARMv8())) {
12434 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12435 }
12436 return Register();
12437}
12438
12439Register fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12440 if (RetVT.SimpleTy != MVT::f64)
12441 return Register();
12442 if ((Subtarget->hasFPARMv8())) {
12443 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12444 }
12445 return Register();
12446}
12447
12448Register fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12449 if (RetVT.SimpleTy != MVT::v4f16)
12450 return Register();
12451 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12452 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12453 }
12454 return Register();
12455}
12456
12457Register fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12458 if (RetVT.SimpleTy != MVT::v8f16)
12459 return Register();
12460 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12461 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12462 }
12463 return Register();
12464}
12465
12466Register fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12467 if (RetVT.SimpleTy != MVT::v2f32)
12468 return Register();
12469 if ((Subtarget->isNeonAvailable())) {
12470 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12471 }
12472 return Register();
12473}
12474
12475Register fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12476 if (RetVT.SimpleTy != MVT::v4f32)
12477 return Register();
12478 if ((Subtarget->isNeonAvailable())) {
12479 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12480 }
12481 return Register();
12482}
12483
12484Register fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12485 if (RetVT.SimpleTy != MVT::v2f64)
12486 return Register();
12487 if ((Subtarget->isNeonAvailable())) {
12488 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12489 }
12490 return Register();
12491}
12492
12493Register fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12494 switch (VT.SimpleTy) {
12495 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12496 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12497 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12498 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12499 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12500 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12501 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12502 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12503 default: return Register();
12504 }
12505}
12506
12507// FastEmit functions for ISD::STRICT_FMAXIMUM.
12508
12509Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12510 if (RetVT.SimpleTy != MVT::f16)
12511 return Register();
12512 if ((Subtarget->hasFullFP16())) {
12513 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12514 }
12515 return Register();
12516}
12517
12518Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12519 if (RetVT.SimpleTy != MVT::f32)
12520 return Register();
12521 if ((Subtarget->hasFPARMv8())) {
12522 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12523 }
12524 return Register();
12525}
12526
12527Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12528 if (RetVT.SimpleTy != MVT::f64)
12529 return Register();
12530 if ((Subtarget->hasFPARMv8())) {
12531 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12532 }
12533 return Register();
12534}
12535
12536Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12537 if (RetVT.SimpleTy != MVT::v4f16)
12538 return Register();
12539 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12540 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12541 }
12542 return Register();
12543}
12544
12545Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12546 if (RetVT.SimpleTy != MVT::v8f16)
12547 return Register();
12548 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12549 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12550 }
12551 return Register();
12552}
12553
12554Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12555 if (RetVT.SimpleTy != MVT::v2f32)
12556 return Register();
12557 if ((Subtarget->isNeonAvailable())) {
12558 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12559 }
12560 return Register();
12561}
12562
12563Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12564 if (RetVT.SimpleTy != MVT::v4f32)
12565 return Register();
12566 if ((Subtarget->isNeonAvailable())) {
12567 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12568 }
12569 return Register();
12570}
12571
12572Register fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12573 if (RetVT.SimpleTy != MVT::v2f64)
12574 return Register();
12575 if ((Subtarget->isNeonAvailable())) {
12576 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12577 }
12578 return Register();
12579}
12580
12581Register fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12582 switch (VT.SimpleTy) {
12583 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12584 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12585 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12586 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12587 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12588 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12589 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12590 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12591 default: return Register();
12592 }
12593}
12594
12595// FastEmit functions for ISD::STRICT_FMAXNUM.
12596
12597Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12598 if (RetVT.SimpleTy != MVT::f16)
12599 return Register();
12600 if ((Subtarget->hasFullFP16())) {
12601 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12602 }
12603 return Register();
12604}
12605
12606Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12607 if (RetVT.SimpleTy != MVT::f32)
12608 return Register();
12609 if ((Subtarget->hasFPARMv8())) {
12610 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12611 }
12612 return Register();
12613}
12614
12615Register fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12616 if (RetVT.SimpleTy != MVT::f64)
12617 return Register();
12618 if ((Subtarget->hasFPARMv8())) {
12619 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12620 }
12621 return Register();
12622}
12623
12624Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12625 if (RetVT.SimpleTy != MVT::v4f16)
12626 return Register();
12627 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12628 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12629 }
12630 return Register();
12631}
12632
12633Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12634 if (RetVT.SimpleTy != MVT::v8f16)
12635 return Register();
12636 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12637 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12638 }
12639 return Register();
12640}
12641
12642Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12643 if (RetVT.SimpleTy != MVT::v2f32)
12644 return Register();
12645 if ((Subtarget->isNeonAvailable())) {
12646 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12647 }
12648 return Register();
12649}
12650
12651Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12652 if (RetVT.SimpleTy != MVT::v4f32)
12653 return Register();
12654 if ((Subtarget->isNeonAvailable())) {
12655 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12656 }
12657 return Register();
12658}
12659
12660Register fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12661 if (RetVT.SimpleTy != MVT::v2f64)
12662 return Register();
12663 if ((Subtarget->isNeonAvailable())) {
12664 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12665 }
12666 return Register();
12667}
12668
12669Register fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12670 switch (VT.SimpleTy) {
12671 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
12672 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
12673 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
12674 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12675 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12676 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12677 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12678 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12679 default: return Register();
12680 }
12681}
12682
12683// FastEmit functions for ISD::STRICT_FMINIMUM.
12684
12685Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12686 if (RetVT.SimpleTy != MVT::f16)
12687 return Register();
12688 if ((Subtarget->hasFullFP16())) {
12689 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12690 }
12691 return Register();
12692}
12693
12694Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12695 if (RetVT.SimpleTy != MVT::f32)
12696 return Register();
12697 if ((Subtarget->hasFPARMv8())) {
12698 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12699 }
12700 return Register();
12701}
12702
12703Register fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12704 if (RetVT.SimpleTy != MVT::f64)
12705 return Register();
12706 if ((Subtarget->hasFPARMv8())) {
12707 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12708 }
12709 return Register();
12710}
12711
12712Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12713 if (RetVT.SimpleTy != MVT::v4f16)
12714 return Register();
12715 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12716 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12717 }
12718 return Register();
12719}
12720
12721Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12722 if (RetVT.SimpleTy != MVT::v8f16)
12723 return Register();
12724 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12725 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12726 }
12727 return Register();
12728}
12729
12730Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12731 if (RetVT.SimpleTy != MVT::v2f32)
12732 return Register();
12733 if ((Subtarget->isNeonAvailable())) {
12734 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12735 }
12736 return Register();
12737}
12738
12739Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12740 if (RetVT.SimpleTy != MVT::v4f32)
12741 return Register();
12742 if ((Subtarget->isNeonAvailable())) {
12743 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12744 }
12745 return Register();
12746}
12747
12748Register fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12749 if (RetVT.SimpleTy != MVT::v2f64)
12750 return Register();
12751 if ((Subtarget->isNeonAvailable())) {
12752 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12753 }
12754 return Register();
12755}
12756
12757Register fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12758 switch (VT.SimpleTy) {
12759 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12760 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12761 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12762 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12763 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12764 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12765 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12766 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12767 default: return Register();
12768 }
12769}
12770
12771// FastEmit functions for ISD::STRICT_FMINNUM.
12772
12773Register fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12774 if (RetVT.SimpleTy != MVT::f16)
12775 return Register();
12776 if ((Subtarget->hasFullFP16())) {
12777 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12778 }
12779 return Register();
12780}
12781
12782Register fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12783 if (RetVT.SimpleTy != MVT::f32)
12784 return Register();
12785 if ((Subtarget->hasFPARMv8())) {
12786 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12787 }
12788 return Register();
12789}
12790
12791Register fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12792 if (RetVT.SimpleTy != MVT::f64)
12793 return Register();
12794 if ((Subtarget->hasFPARMv8())) {
12795 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12796 }
12797 return Register();
12798}
12799
12800Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12801 if (RetVT.SimpleTy != MVT::v4f16)
12802 return Register();
12803 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12804 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12805 }
12806 return Register();
12807}
12808
12809Register fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12810 if (RetVT.SimpleTy != MVT::v8f16)
12811 return Register();
12812 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12813 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12814 }
12815 return Register();
12816}
12817
12818Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12819 if (RetVT.SimpleTy != MVT::v2f32)
12820 return Register();
12821 if ((Subtarget->isNeonAvailable())) {
12822 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12823 }
12824 return Register();
12825}
12826
12827Register fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12828 if (RetVT.SimpleTy != MVT::v4f32)
12829 return Register();
12830 if ((Subtarget->isNeonAvailable())) {
12831 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12832 }
12833 return Register();
12834}
12835
12836Register fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12837 if (RetVT.SimpleTy != MVT::v2f64)
12838 return Register();
12839 if ((Subtarget->isNeonAvailable())) {
12840 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12841 }
12842 return Register();
12843}
12844
12845Register fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12846 switch (VT.SimpleTy) {
12847 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
12848 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
12849 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
12850 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12851 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12852 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12853 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12854 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12855 default: return Register();
12856 }
12857}
12858
12859// FastEmit functions for ISD::STRICT_FMUL.
12860
12861Register fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12862 if (RetVT.SimpleTy != MVT::f16)
12863 return Register();
12864 if ((Subtarget->hasFullFP16())) {
12865 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12866 }
12867 return Register();
12868}
12869
12870Register fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12871 if (RetVT.SimpleTy != MVT::f32)
12872 return Register();
12873 if ((Subtarget->hasFPARMv8())) {
12874 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12875 }
12876 return Register();
12877}
12878
12879Register fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12880 if (RetVT.SimpleTy != MVT::f64)
12881 return Register();
12882 if ((Subtarget->hasFPARMv8())) {
12883 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12884 }
12885 return Register();
12886}
12887
12888Register fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12889 if (RetVT.SimpleTy != MVT::v4f16)
12890 return Register();
12891 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12892 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12893 }
12894 return Register();
12895}
12896
12897Register fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12898 if (RetVT.SimpleTy != MVT::v8f16)
12899 return Register();
12900 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12901 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12902 }
12903 return Register();
12904}
12905
12906Register fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12907 if (RetVT.SimpleTy != MVT::v2f32)
12908 return Register();
12909 if ((Subtarget->isNeonAvailable())) {
12910 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12911 }
12912 return Register();
12913}
12914
12915Register fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
12916 if (RetVT.SimpleTy != MVT::v4f32)
12917 return Register();
12918 if ((Subtarget->isNeonAvailable())) {
12919 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12920 }
12921 return Register();
12922}
12923
12924Register fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
12925 if (RetVT.SimpleTy != MVT::v2f64)
12926 return Register();
12927 if ((Subtarget->isNeonAvailable())) {
12928 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12929 }
12930 return Register();
12931}
12932
12933Register fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
12934 switch (VT.SimpleTy) {
12935 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
12936 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
12937 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
12938 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
12939 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
12940 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
12941 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
12942 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
12943 default: return Register();
12944 }
12945}
12946
12947// FastEmit functions for ISD::STRICT_FSUB.
12948
12949Register fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, Register Op0, Register Op1) {
12950 if (RetVT.SimpleTy != MVT::f16)
12951 return Register();
12952 if ((Subtarget->hasFullFP16())) {
12953 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12954 }
12955 return Register();
12956}
12957
12958Register fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, Register Op0, Register Op1) {
12959 if (RetVT.SimpleTy != MVT::f32)
12960 return Register();
12961 if ((Subtarget->hasFPARMv8())) {
12962 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12963 }
12964 return Register();
12965}
12966
12967Register fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, Register Op0, Register Op1) {
12968 if (RetVT.SimpleTy != MVT::f64)
12969 return Register();
12970 if ((Subtarget->hasFPARMv8())) {
12971 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12972 }
12973 return Register();
12974}
12975
12976Register fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, Register Op0, Register Op1) {
12977 if (RetVT.SimpleTy != MVT::v4f16)
12978 return Register();
12979 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12980 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12981 }
12982 return Register();
12983}
12984
12985Register fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, Register Op0, Register Op1) {
12986 if (RetVT.SimpleTy != MVT::v8f16)
12987 return Register();
12988 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12989 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12990 }
12991 return Register();
12992}
12993
12994Register fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, Register Op0, Register Op1) {
12995 if (RetVT.SimpleTy != MVT::v2f32)
12996 return Register();
12997 if ((Subtarget->isNeonAvailable())) {
12998 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12999 }
13000 return Register();
13001}
13002
13003Register fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, Register Op0, Register Op1) {
13004 if (RetVT.SimpleTy != MVT::v4f32)
13005 return Register();
13006 if ((Subtarget->isNeonAvailable())) {
13007 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13008 }
13009 return Register();
13010}
13011
13012Register fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, Register Op0, Register Op1) {
13013 if (RetVT.SimpleTy != MVT::v2f64)
13014 return Register();
13015 if ((Subtarget->isNeonAvailable())) {
13016 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13017 }
13018 return Register();
13019}
13020
13021Register fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13022 switch (VT.SimpleTy) {
13023 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13024 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13025 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13026 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13027 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13028 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13029 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13030 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13031 default: return Register();
13032 }
13033}
13034
13035// FastEmit functions for ISD::SUB.
13036
13037Register fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13038 if (RetVT.SimpleTy != MVT::i32)
13039 return Register();
13040 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13041}
13042
13043Register fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13044 if (RetVT.SimpleTy != MVT::i64)
13045 return Register();
13046 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13047}
13048
13049Register fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13050 if (RetVT.SimpleTy != MVT::v8i8)
13051 return Register();
13052 if ((Subtarget->isNeonAvailable())) {
13053 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13054 }
13055 return Register();
13056}
13057
13058Register fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13059 if (RetVT.SimpleTy != MVT::v16i8)
13060 return Register();
13061 if ((Subtarget->isNeonAvailable())) {
13062 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13063 }
13064 return Register();
13065}
13066
13067Register fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13068 if (RetVT.SimpleTy != MVT::v4i16)
13069 return Register();
13070 if ((Subtarget->isNeonAvailable())) {
13071 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13072 }
13073 return Register();
13074}
13075
13076Register fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13077 if (RetVT.SimpleTy != MVT::v8i16)
13078 return Register();
13079 if ((Subtarget->isNeonAvailable())) {
13080 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13081 }
13082 return Register();
13083}
13084
13085Register fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13086 if (RetVT.SimpleTy != MVT::v2i32)
13087 return Register();
13088 if ((Subtarget->isNeonAvailable())) {
13089 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13090 }
13091 return Register();
13092}
13093
13094Register fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13095 if (RetVT.SimpleTy != MVT::v4i32)
13096 return Register();
13097 if ((Subtarget->isNeonAvailable())) {
13098 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13099 }
13100 return Register();
13101}
13102
13103Register fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13104 if (RetVT.SimpleTy != MVT::v1i64)
13105 return Register();
13106 if ((Subtarget->isNeonAvailable())) {
13107 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13108 }
13109 return Register();
13110}
13111
13112Register fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13113 if (RetVT.SimpleTy != MVT::v2i64)
13114 return Register();
13115 if ((Subtarget->isNeonAvailable())) {
13116 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13117 }
13118 return Register();
13119}
13120
13121Register fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13122 if (RetVT.SimpleTy != MVT::nxv16i8)
13123 return Register();
13124 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13125 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13126 }
13127 return Register();
13128}
13129
13130Register fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13131 if (RetVT.SimpleTy != MVT::nxv8i16)
13132 return Register();
13133 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13134 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13135 }
13136 return Register();
13137}
13138
13139Register fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13140 if (RetVT.SimpleTy != MVT::nxv4i32)
13141 return Register();
13142 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13143 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13144 }
13145 return Register();
13146}
13147
13148Register fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13149 if (RetVT.SimpleTy != MVT::nxv2i64)
13150 return Register();
13151 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13152 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13153 }
13154 return Register();
13155}
13156
13157Register fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13158 switch (VT.SimpleTy) {
13159 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13160 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13161 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13162 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13163 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13164 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13165 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13166 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13167 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13168 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13169 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13170 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13171 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13172 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13173 default: return Register();
13174 }
13175}
13176
13177// FastEmit functions for ISD::UADDSAT.
13178
13179Register fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13180 if (RetVT.SimpleTy != MVT::v8i8)
13181 return Register();
13182 if ((Subtarget->isNeonAvailable())) {
13183 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13184 }
13185 return Register();
13186}
13187
13188Register fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13189 if (RetVT.SimpleTy != MVT::v16i8)
13190 return Register();
13191 if ((Subtarget->isNeonAvailable())) {
13192 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13193 }
13194 return Register();
13195}
13196
13197Register fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13198 if (RetVT.SimpleTy != MVT::v4i16)
13199 return Register();
13200 if ((Subtarget->isNeonAvailable())) {
13201 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13202 }
13203 return Register();
13204}
13205
13206Register fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13207 if (RetVT.SimpleTy != MVT::v8i16)
13208 return Register();
13209 if ((Subtarget->isNeonAvailable())) {
13210 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13211 }
13212 return Register();
13213}
13214
13215Register fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13216 if (RetVT.SimpleTy != MVT::v2i32)
13217 return Register();
13218 if ((Subtarget->isNeonAvailable())) {
13219 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13220 }
13221 return Register();
13222}
13223
13224Register fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13225 if (RetVT.SimpleTy != MVT::v4i32)
13226 return Register();
13227 if ((Subtarget->isNeonAvailable())) {
13228 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13229 }
13230 return Register();
13231}
13232
13233Register fastEmit_ISD_UADDSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13234 if (RetVT.SimpleTy != MVT::v1i64)
13235 return Register();
13236 if ((Subtarget->isNeonAvailable())) {
13237 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13238 }
13239 return Register();
13240}
13241
13242Register fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13243 if (RetVT.SimpleTy != MVT::v2i64)
13244 return Register();
13245 if ((Subtarget->isNeonAvailable())) {
13246 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13247 }
13248 return Register();
13249}
13250
13251Register fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13252 if (RetVT.SimpleTy != MVT::nxv16i8)
13253 return Register();
13254 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13255 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13256 }
13257 return Register();
13258}
13259
13260Register fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13261 if (RetVT.SimpleTy != MVT::nxv8i16)
13262 return Register();
13263 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13264 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13265 }
13266 return Register();
13267}
13268
13269Register fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13270 if (RetVT.SimpleTy != MVT::nxv4i32)
13271 return Register();
13272 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13273 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13274 }
13275 return Register();
13276}
13277
13278Register fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13279 if (RetVT.SimpleTy != MVT::nxv2i64)
13280 return Register();
13281 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13282 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13283 }
13284 return Register();
13285}
13286
13287Register fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13288 switch (VT.SimpleTy) {
13289 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13290 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13291 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13292 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13293 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13294 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13295 case MVT::v1i64: return fastEmit_ISD_UADDSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13296 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13297 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13298 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13299 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13300 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13301 default: return Register();
13302 }
13303}
13304
13305// FastEmit functions for ISD::UDIV.
13306
13307Register fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13308 if (RetVT.SimpleTy != MVT::i32)
13309 return Register();
13310 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13311}
13312
13313Register fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13314 if (RetVT.SimpleTy != MVT::i64)
13315 return Register();
13316 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13317}
13318
13319Register fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13320 switch (VT.SimpleTy) {
13321 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13322 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13323 default: return Register();
13324 }
13325}
13326
13327// FastEmit functions for ISD::UMAX.
13328
13329Register fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13330 if (RetVT.SimpleTy != MVT::i32)
13331 return Register();
13332 if ((Subtarget->hasCSSC())) {
13333 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13334 }
13335 return Register();
13336}
13337
13338Register fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13339 if (RetVT.SimpleTy != MVT::i64)
13340 return Register();
13341 if ((Subtarget->hasCSSC())) {
13342 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13343 }
13344 return Register();
13345}
13346
13347Register fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13348 if (RetVT.SimpleTy != MVT::v8i8)
13349 return Register();
13350 if ((Subtarget->isNeonAvailable())) {
13351 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13352 }
13353 return Register();
13354}
13355
13356Register fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13357 if (RetVT.SimpleTy != MVT::v16i8)
13358 return Register();
13359 if ((Subtarget->isNeonAvailable())) {
13360 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13361 }
13362 return Register();
13363}
13364
13365Register fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13366 if (RetVT.SimpleTy != MVT::v4i16)
13367 return Register();
13368 if ((Subtarget->isNeonAvailable())) {
13369 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13370 }
13371 return Register();
13372}
13373
13374Register fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13375 if (RetVT.SimpleTy != MVT::v8i16)
13376 return Register();
13377 if ((Subtarget->isNeonAvailable())) {
13378 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13379 }
13380 return Register();
13381}
13382
13383Register fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13384 if (RetVT.SimpleTy != MVT::v2i32)
13385 return Register();
13386 if ((Subtarget->isNeonAvailable())) {
13387 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13388 }
13389 return Register();
13390}
13391
13392Register fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13393 if (RetVT.SimpleTy != MVT::v4i32)
13394 return Register();
13395 if ((Subtarget->isNeonAvailable())) {
13396 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13397 }
13398 return Register();
13399}
13400
13401Register fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13402 switch (VT.SimpleTy) {
13403 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13404 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13405 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13406 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13407 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13408 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13409 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13410 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13411 default: return Register();
13412 }
13413}
13414
13415// FastEmit functions for ISD::UMIN.
13416
13417Register fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13418 if (RetVT.SimpleTy != MVT::i32)
13419 return Register();
13420 if ((Subtarget->hasCSSC())) {
13421 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13422 }
13423 return Register();
13424}
13425
13426Register fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13427 if (RetVT.SimpleTy != MVT::i64)
13428 return Register();
13429 if ((Subtarget->hasCSSC())) {
13430 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13431 }
13432 return Register();
13433}
13434
13435Register fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13436 if (RetVT.SimpleTy != MVT::v8i8)
13437 return Register();
13438 if ((Subtarget->isNeonAvailable())) {
13439 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13440 }
13441 return Register();
13442}
13443
13444Register fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13445 if (RetVT.SimpleTy != MVT::v16i8)
13446 return Register();
13447 if ((Subtarget->isNeonAvailable())) {
13448 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13449 }
13450 return Register();
13451}
13452
13453Register fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13454 if (RetVT.SimpleTy != MVT::v4i16)
13455 return Register();
13456 if ((Subtarget->isNeonAvailable())) {
13457 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13458 }
13459 return Register();
13460}
13461
13462Register fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13463 if (RetVT.SimpleTy != MVT::v8i16)
13464 return Register();
13465 if ((Subtarget->isNeonAvailable())) {
13466 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13467 }
13468 return Register();
13469}
13470
13471Register fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13472 if (RetVT.SimpleTy != MVT::v2i32)
13473 return Register();
13474 if ((Subtarget->isNeonAvailable())) {
13475 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13476 }
13477 return Register();
13478}
13479
13480Register fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13481 if (RetVT.SimpleTy != MVT::v4i32)
13482 return Register();
13483 if ((Subtarget->isNeonAvailable())) {
13484 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13485 }
13486 return Register();
13487}
13488
13489Register fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13490 switch (VT.SimpleTy) {
13491 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13492 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13493 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13494 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13495 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13496 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13497 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13498 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13499 default: return Register();
13500 }
13501}
13502
13503// FastEmit functions for ISD::USUBSAT.
13504
13505Register fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13506 if (RetVT.SimpleTy != MVT::v8i8)
13507 return Register();
13508 if ((Subtarget->isNeonAvailable())) {
13509 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13510 }
13511 return Register();
13512}
13513
13514Register fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13515 if (RetVT.SimpleTy != MVT::v16i8)
13516 return Register();
13517 if ((Subtarget->isNeonAvailable())) {
13518 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13519 }
13520 return Register();
13521}
13522
13523Register fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13524 if (RetVT.SimpleTy != MVT::v4i16)
13525 return Register();
13526 if ((Subtarget->isNeonAvailable())) {
13527 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13528 }
13529 return Register();
13530}
13531
13532Register fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13533 if (RetVT.SimpleTy != MVT::v8i16)
13534 return Register();
13535 if ((Subtarget->isNeonAvailable())) {
13536 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13537 }
13538 return Register();
13539}
13540
13541Register fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13542 if (RetVT.SimpleTy != MVT::v2i32)
13543 return Register();
13544 if ((Subtarget->isNeonAvailable())) {
13545 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13546 }
13547 return Register();
13548}
13549
13550Register fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13551 if (RetVT.SimpleTy != MVT::v4i32)
13552 return Register();
13553 if ((Subtarget->isNeonAvailable())) {
13554 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13555 }
13556 return Register();
13557}
13558
13559Register fastEmit_ISD_USUBSAT_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13560 if (RetVT.SimpleTy != MVT::v1i64)
13561 return Register();
13562 if ((Subtarget->isNeonAvailable())) {
13563 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13564 }
13565 return Register();
13566}
13567
13568Register fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13569 if (RetVT.SimpleTy != MVT::v2i64)
13570 return Register();
13571 if ((Subtarget->isNeonAvailable())) {
13572 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13573 }
13574 return Register();
13575}
13576
13577Register fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13578 if (RetVT.SimpleTy != MVT::nxv16i8)
13579 return Register();
13580 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13581 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13582 }
13583 return Register();
13584}
13585
13586Register fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13587 if (RetVT.SimpleTy != MVT::nxv8i16)
13588 return Register();
13589 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13590 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13591 }
13592 return Register();
13593}
13594
13595Register fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13596 if (RetVT.SimpleTy != MVT::nxv4i32)
13597 return Register();
13598 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13599 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13600 }
13601 return Register();
13602}
13603
13604Register fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13605 if (RetVT.SimpleTy != MVT::nxv2i64)
13606 return Register();
13607 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13608 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13609 }
13610 return Register();
13611}
13612
13613Register fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13614 switch (VT.SimpleTy) {
13615 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13616 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13617 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13618 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13619 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13620 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13621 case MVT::v1i64: return fastEmit_ISD_USUBSAT_MVT_v1i64_rr(RetVT, Op0, Op1);
13622 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13623 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13624 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13625 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13626 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13627 default: return Register();
13628 }
13629}
13630
13631// FastEmit functions for ISD::XOR.
13632
13633Register fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, Register Op0, Register Op1) {
13634 if (RetVT.SimpleTy != MVT::i32)
13635 return Register();
13636 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13637}
13638
13639Register fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, Register Op0, Register Op1) {
13640 if (RetVT.SimpleTy != MVT::i64)
13641 return Register();
13642 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13643}
13644
13645Register fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, Register Op0, Register Op1) {
13646 if (RetVT.SimpleTy != MVT::v8i8)
13647 return Register();
13648 if ((Subtarget->isNeonAvailable())) {
13649 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13650 }
13651 return Register();
13652}
13653
13654Register fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13655 if (RetVT.SimpleTy != MVT::v16i8)
13656 return Register();
13657 if ((Subtarget->isNeonAvailable())) {
13658 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13659 }
13660 return Register();
13661}
13662
13663Register fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, Register Op0, Register Op1) {
13664 if (RetVT.SimpleTy != MVT::v4i16)
13665 return Register();
13666 if ((Subtarget->isNeonAvailable())) {
13667 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13668 }
13669 return Register();
13670}
13671
13672Register fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13673 if (RetVT.SimpleTy != MVT::v8i16)
13674 return Register();
13675 if ((Subtarget->isNeonAvailable())) {
13676 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13677 }
13678 return Register();
13679}
13680
13681Register fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, Register Op0, Register Op1) {
13682 if (RetVT.SimpleTy != MVT::v2i32)
13683 return Register();
13684 if ((Subtarget->isNeonAvailable())) {
13685 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13686 }
13687 return Register();
13688}
13689
13690Register fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13691 if (RetVT.SimpleTy != MVT::v4i32)
13692 return Register();
13693 if ((Subtarget->isNeonAvailable())) {
13694 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13695 }
13696 return Register();
13697}
13698
13699Register fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, Register Op0, Register Op1) {
13700 if (RetVT.SimpleTy != MVT::v1i64)
13701 return Register();
13702 if ((Subtarget->isNeonAvailable())) {
13703 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13704 }
13705 return Register();
13706}
13707
13708Register fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13709 if (RetVT.SimpleTy != MVT::v2i64)
13710 return Register();
13711 if ((Subtarget->isNeonAvailable())) {
13712 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13713 }
13714 return Register();
13715}
13716
13717Register fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, Register Op0, Register Op1) {
13718 if (RetVT.SimpleTy != MVT::nxv16i8)
13719 return Register();
13720 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13721 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13722 }
13723 return Register();
13724}
13725
13726Register fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, Register Op0, Register Op1) {
13727 if (RetVT.SimpleTy != MVT::nxv8i16)
13728 return Register();
13729 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13730 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13731 }
13732 return Register();
13733}
13734
13735Register fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, Register Op0, Register Op1) {
13736 if (RetVT.SimpleTy != MVT::nxv4i32)
13737 return Register();
13738 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13739 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13740 }
13741 return Register();
13742}
13743
13744Register fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, Register Op0, Register Op1) {
13745 if (RetVT.SimpleTy != MVT::nxv2i64)
13746 return Register();
13747 if ((Subtarget->isSVEorStreamingSVEAvailable())) {
13748 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
13749 }
13750 return Register();
13751}
13752
13753Register fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, Register Op0, Register Op1) {
13754 switch (VT.SimpleTy) {
13755 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
13756 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
13757 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
13758 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
13759 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
13760 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
13761 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
13762 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
13763 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
13764 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
13765 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13766 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13767 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13768 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13769 default: return Register();
13770 }
13771}
13772
13773// Top-level FastEmit function.
13774
13775Register fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, Register Op1) override {
13776 switch (Opcode) {
13777 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
13778 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
13779 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
13780 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
13781 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
13782 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
13783 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
13784 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
13785 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
13786 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
13787 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
13788 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
13789 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
13790 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
13791 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
13792 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
13793 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
13794 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
13795 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
13796 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
13797 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
13798 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
13799 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
13800 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
13801 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
13802 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
13803 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
13804 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
13805 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
13806 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
13807 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
13808 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
13809 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
13810 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
13811 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
13812 case ISD::FMAXNUM_IEEE: return fastEmit_ISD_FMAXNUM_IEEE_rr(VT, RetVT, Op0, Op1);
13813 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
13814 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
13815 case ISD::FMINNUM_IEEE: return fastEmit_ISD_FMINNUM_IEEE_rr(VT, RetVT, Op0, Op1);
13816 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
13817 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
13818 case ISD::GET_ACTIVE_LANE_MASK: return fastEmit_ISD_GET_ACTIVE_LANE_MASK_rr(VT, RetVT, Op0, Op1);
13819 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
13820 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
13821 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
13822 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
13823 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
13824 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
13825 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
13826 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
13827 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
13828 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
13829 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
13830 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
13831 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
13832 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
13833 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
13834 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
13835 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
13836 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
13837 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
13838 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
13839 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
13840 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
13841 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
13842 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
13843 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
13844 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
13845 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
13846 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
13847 default: return Register();
13848 }
13849}
13850
13851// FastEmit functions for AArch64ISD::DUPLANE64.
13852
13853Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
13854 if (RetVT.SimpleTy != MVT::v2i64)
13855 return Register();
13856 if ((Subtarget->isNeonAvailable())) {
13857 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
13858 }
13859 return Register();
13860}
13861
13862Register fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
13863 if (RetVT.SimpleTy != MVT::v2f64)
13864 return Register();
13865 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
13866}
13867
13868Register fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
13869 switch (VT.SimpleTy) {
13870 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
13871 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
13872 default: return Register();
13873 }
13874}
13875
13876// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
13877
13878Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
13879 if (RetVT.SimpleTy != MVT::i64)
13880 return Register();
13881 if ((Subtarget->isNeonAvailable())) {
13882 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
13883 }
13884 return Register();
13885}
13886
13887Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, Register Op0, uint64_t imm1) {
13888 if (RetVT.SimpleTy != MVT::f64)
13889 return Register();
13890 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
13891}
13892
13893Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
13894 switch (VT.SimpleTy) {
13895 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
13896 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
13897 default: return Register();
13898 }
13899}
13900
13901// Top-level FastEmit function.
13902
13903Register fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
13904 switch (Opcode) {
13905 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
13906 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
13907 default: return Register();
13908 }
13909}
13910
13911// FastEmit functions for AArch64ISD::DUPLANE32.
13912
13913Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
13914 if ((Subtarget->isNeonAvailable())) {
13915 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
13916 }
13917 return Register();
13918}
13919
13920Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
13921 if ((Subtarget->isNeonAvailable())) {
13922 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
13923 }
13924 return Register();
13925}
13926
13927Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
13928switch (RetVT.SimpleTy) {
13929 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
13930 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
13931 default: return Register();
13932}
13933}
13934
13935Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
13936 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
13937}
13938
13939Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Register Op0, uint64_t imm1) {
13940 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
13941}
13942
13943Register fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
13944switch (RetVT.SimpleTy) {
13945 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
13946 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
13947 default: return Register();
13948}
13949}
13950
13951Register fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
13952 switch (VT.SimpleTy) {
13953 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
13954 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
13955 default: return Register();
13956 }
13957}
13958
13959// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
13960
13961Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
13962 if (RetVT.SimpleTy != MVT::i32)
13963 return Register();
13964 if ((Subtarget->isNeonAvailable())) {
13965 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
13966 }
13967 return Register();
13968}
13969
13970Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, Register Op0, uint64_t imm1) {
13971 if (RetVT.SimpleTy != MVT::f32)
13972 return Register();
13973 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
13974}
13975
13976Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
13977 switch (VT.SimpleTy) {
13978 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
13979 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
13980 default: return Register();
13981 }
13982}
13983
13984// Top-level FastEmit function.
13985
13986Register fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
13987 switch (Opcode) {
13988 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
13989 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
13990 default: return Register();
13991 }
13992}
13993
13994// FastEmit functions for AArch64ISD::DUPLANE16.
13995
13996Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
13997 if ((Subtarget->isNeonAvailable())) {
13998 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
13999 }
14000 return Register();
14001}
14002
14003Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14004 if ((Subtarget->isNeonAvailable())) {
14005 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14006 }
14007 return Register();
14008}
14009
14010Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14011switch (RetVT.SimpleTy) {
14012 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14013 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14014 default: return Register();
14015}
14016}
14017
14018Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14019 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14020}
14021
14022Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14023 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14024}
14025
14026Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14027switch (RetVT.SimpleTy) {
14028 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14029 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14030 default: return Register();
14031}
14032}
14033
14034Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14035 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14036}
14037
14038Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Register Op0, uint64_t imm1) {
14039 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14040}
14041
14042Register fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14043switch (RetVT.SimpleTy) {
14044 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14045 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14046 default: return Register();
14047}
14048}
14049
14050Register fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14051 switch (VT.SimpleTy) {
14052 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14053 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14054 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14055 default: return Register();
14056 }
14057}
14058
14059// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14060
14061Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14062 if (RetVT.SimpleTy != MVT::i32)
14063 return Register();
14064 if ((Subtarget->isNeonAvailable())) {
14065 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14066 }
14067 return Register();
14068}
14069
14070Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14071 if (RetVT.SimpleTy != MVT::f16)
14072 return Register();
14073 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14074}
14075
14076Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, Register Op0, uint64_t imm1) {
14077 if (RetVT.SimpleTy != MVT::bf16)
14078 return Register();
14079 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14080}
14081
14082Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14083 switch (VT.SimpleTy) {
14084 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14085 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14086 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14087 default: return Register();
14088 }
14089}
14090
14091// Top-level FastEmit function.
14092
14093Register fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14094 switch (Opcode) {
14095 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14096 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14097 default: return Register();
14098 }
14099}
14100
14101// FastEmit functions for AArch64ISD::DUPLANE8.
14102
14103Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14104 if ((Subtarget->isNeonAvailable())) {
14105 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14106 }
14107 return Register();
14108}
14109
14110Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Register Op0, uint64_t imm1) {
14111 if ((Subtarget->isNeonAvailable())) {
14112 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14113 }
14114 return Register();
14115}
14116
14117Register fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14118switch (RetVT.SimpleTy) {
14119 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14120 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14121 default: return Register();
14122}
14123}
14124
14125Register fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14126 switch (VT.SimpleTy) {
14127 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14128 default: return Register();
14129 }
14130}
14131
14132// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14133
14134Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, Register Op0, uint64_t imm1) {
14135 if (RetVT.SimpleTy != MVT::i32)
14136 return Register();
14137 if ((Subtarget->isNeonAvailable())) {
14138 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14139 }
14140 return Register();
14141}
14142
14143Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14144 switch (VT.SimpleTy) {
14145 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14146 default: return Register();
14147 }
14148}
14149
14150// Top-level FastEmit function.
14151
14152Register fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14153 switch (Opcode) {
14154 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14155 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14156 default: return Register();
14157 }
14158}
14159
14160// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14161
14162Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14163 if (RetVT.SimpleTy != MVT::i32)
14164 return Register();
14165 if ((Subtarget->hasNEON())) {
14166 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14167 }
14168 return Register();
14169}
14170
14171Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14172 if (RetVT.SimpleTy != MVT::i32)
14173 return Register();
14174 if ((Subtarget->hasNEON())) {
14175 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14176 }
14177 return Register();
14178}
14179
14180Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14181 if (RetVT.SimpleTy != MVT::i32)
14182 return Register();
14183 if ((Subtarget->hasNEON())) {
14184 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14185 }
14186 return Register();
14187}
14188
14189Register fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, Register Op0, uint64_t imm1) {
14190 if (RetVT.SimpleTy != MVT::i64)
14191 return Register();
14192 if ((Subtarget->hasNEON())) {
14193 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14194 }
14195 return Register();
14196}
14197
14198Register fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14199 switch (VT.SimpleTy) {
14200 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14201 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14202 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14203 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14204 default: return Register();
14205 }
14206}
14207
14208// Top-level FastEmit function.
14209
14210Register fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14211 switch (Opcode) {
14212 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14213 default: return Register();
14214 }
14215}
14216
14217// FastEmit functions for AArch64ISD::SQSHLU_I.
14218
14219Register fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14220 if (RetVT.SimpleTy != MVT::i64)
14221 return Register();
14222 if ((Subtarget->isNeonAvailable())) {
14223 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14224 }
14225 return Register();
14226}
14227
14228Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14229 if (RetVT.SimpleTy != MVT::v1i64)
14230 return Register();
14231 if ((Subtarget->isNeonAvailable())) {
14232 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14233 }
14234 return Register();
14235}
14236
14237Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14238 if (RetVT.SimpleTy != MVT::v2i64)
14239 return Register();
14240 if ((Subtarget->isNeonAvailable())) {
14241 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14242 }
14243 return Register();
14244}
14245
14246Register fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14247 switch (VT.SimpleTy) {
14248 case MVT::i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14249 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14250 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14251 default: return Register();
14252 }
14253}
14254
14255// FastEmit functions for AArch64ISD::SQSHL_I.
14256
14257Register fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14258 if (RetVT.SimpleTy != MVT::i64)
14259 return Register();
14260 if ((Subtarget->isNeonAvailable())) {
14261 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14262 }
14263 return Register();
14264}
14265
14266Register fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14267 if (RetVT.SimpleTy != MVT::v1i64)
14268 return Register();
14269 if ((Subtarget->isNeonAvailable())) {
14270 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14271 }
14272 return Register();
14273}
14274
14275Register fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14276 if (RetVT.SimpleTy != MVT::v2i64)
14277 return Register();
14278 if ((Subtarget->isNeonAvailable())) {
14279 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14280 }
14281 return Register();
14282}
14283
14284Register fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14285 switch (VT.SimpleTy) {
14286 case MVT::i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14287 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14288 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14289 default: return Register();
14290 }
14291}
14292
14293// FastEmit functions for AArch64ISD::UQSHL_I.
14294
14295Register fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14296 if (RetVT.SimpleTy != MVT::i64)
14297 return Register();
14298 if ((Subtarget->isNeonAvailable())) {
14299 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14300 }
14301 return Register();
14302}
14303
14304Register fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14305 if (RetVT.SimpleTy != MVT::v1i64)
14306 return Register();
14307 if ((Subtarget->isNeonAvailable())) {
14308 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14309 }
14310 return Register();
14311}
14312
14313Register fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14314 if (RetVT.SimpleTy != MVT::v2i64)
14315 return Register();
14316 if ((Subtarget->isNeonAvailable())) {
14317 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14318 }
14319 return Register();
14320}
14321
14322Register fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14323 switch (VT.SimpleTy) {
14324 case MVT::i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14325 case MVT::v1i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14326 case MVT::v2i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14327 default: return Register();
14328 }
14329}
14330
14331// FastEmit functions for AArch64ISD::VSHL.
14332
14333Register fastEmit_AArch64ISD_VSHL_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14334 if (RetVT.SimpleTy != MVT::i64)
14335 return Register();
14336 if ((Subtarget->isNeonAvailable())) {
14337 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14338 }
14339 return Register();
14340}
14341
14342Register fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14343 if (RetVT.SimpleTy != MVT::v1i64)
14344 return Register();
14345 if ((Subtarget->isNeonAvailable())) {
14346 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14347 }
14348 return Register();
14349}
14350
14351Register fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, Register Op0, uint64_t imm1) {
14352 if (RetVT.SimpleTy != MVT::v2i64)
14353 return Register();
14354 if ((Subtarget->isNeonAvailable())) {
14355 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14356 }
14357 return Register();
14358}
14359
14360Register fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14361 switch (VT.SimpleTy) {
14362 case MVT::i64: return fastEmit_AArch64ISD_VSHL_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14363 case MVT::v1i64: return fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14364 case MVT::v2i64: return fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14365 default: return Register();
14366 }
14367}
14368
14369// Top-level FastEmit function.
14370
14371Register fastEmit_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14372 switch (Opcode) {
14373 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14374 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14375 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14376 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14377 default: return Register();
14378 }
14379}
14380
14381// FastEmit functions for AArch64ISD::SQSHLU_I.
14382
14383Register fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14384 if (RetVT.SimpleTy != MVT::i32)
14385 return Register();
14386 if ((Subtarget->isNeonAvailable())) {
14387 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUs, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14388 }
14389 return Register();
14390}
14391
14392Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14393 if (RetVT.SimpleTy != MVT::v2i32)
14394 return Register();
14395 if ((Subtarget->isNeonAvailable())) {
14396 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14397 }
14398 return Register();
14399}
14400
14401Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14402 if (RetVT.SimpleTy != MVT::v4i32)
14403 return Register();
14404 if ((Subtarget->isNeonAvailable())) {
14405 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14406 }
14407 return Register();
14408}
14409
14410Register fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14411 switch (VT.SimpleTy) {
14412 case MVT::i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14413 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14414 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14415 default: return Register();
14416 }
14417}
14418
14419// FastEmit functions for AArch64ISD::SQSHL_I.
14420
14421Register fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14422 if (RetVT.SimpleTy != MVT::i32)
14423 return Register();
14424 if ((Subtarget->isNeonAvailable())) {
14425 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLs, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14426 }
14427 return Register();
14428}
14429
14430Register fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14431 if (RetVT.SimpleTy != MVT::v2i32)
14432 return Register();
14433 if ((Subtarget->isNeonAvailable())) {
14434 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14435 }
14436 return Register();
14437}
14438
14439Register fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14440 if (RetVT.SimpleTy != MVT::v4i32)
14441 return Register();
14442 if ((Subtarget->isNeonAvailable())) {
14443 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14444 }
14445 return Register();
14446}
14447
14448Register fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14449 switch (VT.SimpleTy) {
14450 case MVT::i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14451 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14452 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14453 default: return Register();
14454 }
14455}
14456
14457// FastEmit functions for AArch64ISD::UQSHL_I.
14458
14459Register fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14460 if (RetVT.SimpleTy != MVT::i32)
14461 return Register();
14462 if ((Subtarget->isNeonAvailable())) {
14463 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLs, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14464 }
14465 return Register();
14466}
14467
14468Register fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14469 if (RetVT.SimpleTy != MVT::v2i32)
14470 return Register();
14471 if ((Subtarget->isNeonAvailable())) {
14472 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14473 }
14474 return Register();
14475}
14476
14477Register fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14478 if (RetVT.SimpleTy != MVT::v4i32)
14479 return Register();
14480 if ((Subtarget->isNeonAvailable())) {
14481 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14482 }
14483 return Register();
14484}
14485
14486Register fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14487 switch (VT.SimpleTy) {
14488 case MVT::i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14489 case MVT::v2i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14490 case MVT::v4i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14491 default: return Register();
14492 }
14493}
14494
14495// FastEmit functions for AArch64ISD::VSHL.
14496
14497Register fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14498 if (RetVT.SimpleTy != MVT::v2i32)
14499 return Register();
14500 if ((Subtarget->isNeonAvailable())) {
14501 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14502 }
14503 return Register();
14504}
14505
14506Register fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, Register Op0, uint64_t imm1) {
14507 if (RetVT.SimpleTy != MVT::v4i32)
14508 return Register();
14509 if ((Subtarget->isNeonAvailable())) {
14510 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14511 }
14512 return Register();
14513}
14514
14515Register fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14516 switch (VT.SimpleTy) {
14517 case MVT::v2i32: return fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14518 case MVT::v4i32: return fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14519 default: return Register();
14520 }
14521}
14522
14523// Top-level FastEmit function.
14524
14525Register fastEmit_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14526 switch (Opcode) {
14527 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14528 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14529 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14530 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14531 default: return Register();
14532 }
14533}
14534
14535// FastEmit functions for AArch64ISD::SRSHR_I.
14536
14537Register fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14538 if (RetVT.SimpleTy != MVT::i64)
14539 return Register();
14540 if ((Subtarget->isNeonAvailable())) {
14541 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14542 }
14543 return Register();
14544}
14545
14546Register fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14547 if (RetVT.SimpleTy != MVT::v1i64)
14548 return Register();
14549 if ((Subtarget->isNeonAvailable())) {
14550 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14551 }
14552 return Register();
14553}
14554
14555Register fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14556 if (RetVT.SimpleTy != MVT::v2i64)
14557 return Register();
14558 if ((Subtarget->isNeonAvailable())) {
14559 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14560 }
14561 return Register();
14562}
14563
14564Register fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14565 switch (VT.SimpleTy) {
14566 case MVT::i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14567 case MVT::v1i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14568 case MVT::v2i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14569 default: return Register();
14570 }
14571}
14572
14573// FastEmit functions for AArch64ISD::URSHR_I.
14574
14575Register fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14576 if (RetVT.SimpleTy != MVT::i64)
14577 return Register();
14578 if ((Subtarget->isNeonAvailable())) {
14579 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14580 }
14581 return Register();
14582}
14583
14584Register fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14585 if (RetVT.SimpleTy != MVT::v1i64)
14586 return Register();
14587 if ((Subtarget->isNeonAvailable())) {
14588 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14589 }
14590 return Register();
14591}
14592
14593Register fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14594 if (RetVT.SimpleTy != MVT::v2i64)
14595 return Register();
14596 if ((Subtarget->isNeonAvailable())) {
14597 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14598 }
14599 return Register();
14600}
14601
14602Register fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14603 switch (VT.SimpleTy) {
14604 case MVT::i64: return fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14605 case MVT::v1i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14606 case MVT::v2i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14607 default: return Register();
14608 }
14609}
14610
14611// FastEmit functions for AArch64ISD::VASHR.
14612
14613Register fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14614 if (RetVT.SimpleTy != MVT::i64)
14615 return Register();
14616 if ((Subtarget->isNeonAvailable())) {
14617 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14618 }
14619 return Register();
14620}
14621
14622Register fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14623 if (RetVT.SimpleTy != MVT::v1i64)
14624 return Register();
14625 if ((Subtarget->isNeonAvailable())) {
14626 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14627 }
14628 return Register();
14629}
14630
14631Register fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14632 if (RetVT.SimpleTy != MVT::v2i64)
14633 return Register();
14634 if ((Subtarget->isNeonAvailable())) {
14635 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14636 }
14637 return Register();
14638}
14639
14640Register fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14641 switch (VT.SimpleTy) {
14642 case MVT::i64: return fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14643 case MVT::v1i64: return fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14644 case MVT::v2i64: return fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14645 default: return Register();
14646 }
14647}
14648
14649// FastEmit functions for AArch64ISD::VLSHR.
14650
14651Register fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14652 if (RetVT.SimpleTy != MVT::i64)
14653 return Register();
14654 if ((Subtarget->isNeonAvailable())) {
14655 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14656 }
14657 return Register();
14658}
14659
14660Register fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14661 if (RetVT.SimpleTy != MVT::v1i64)
14662 return Register();
14663 if ((Subtarget->isNeonAvailable())) {
14664 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14665 }
14666 return Register();
14667}
14668
14669Register fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, Register Op0, uint64_t imm1) {
14670 if (RetVT.SimpleTy != MVT::v2i64)
14671 return Register();
14672 if ((Subtarget->isNeonAvailable())) {
14673 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14674 }
14675 return Register();
14676}
14677
14678Register fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14679 switch (VT.SimpleTy) {
14680 case MVT::i64: return fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14681 case MVT::v1i64: return fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14682 case MVT::v2i64: return fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14683 default: return Register();
14684 }
14685}
14686
14687// Top-level FastEmit function.
14688
14689Register fastEmit_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14690 switch (Opcode) {
14691 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14692 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14693 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14694 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14695 default: return Register();
14696 }
14697}
14698
14699// FastEmit functions for AArch64ISD::SQSHLU_I.
14700
14701Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14702 if (RetVT.SimpleTy != MVT::v8i8)
14703 return Register();
14704 if ((Subtarget->isNeonAvailable())) {
14705 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14706 }
14707 return Register();
14708}
14709
14710Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14711 if (RetVT.SimpleTy != MVT::v16i8)
14712 return Register();
14713 if ((Subtarget->isNeonAvailable())) {
14714 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14715 }
14716 return Register();
14717}
14718
14719Register fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14720 switch (VT.SimpleTy) {
14721 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14722 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14723 default: return Register();
14724 }
14725}
14726
14727// FastEmit functions for AArch64ISD::SQSHL_I.
14728
14729Register fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14730 if (RetVT.SimpleTy != MVT::v8i8)
14731 return Register();
14732 if ((Subtarget->isNeonAvailable())) {
14733 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14734 }
14735 return Register();
14736}
14737
14738Register fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14739 if (RetVT.SimpleTy != MVT::v16i8)
14740 return Register();
14741 if ((Subtarget->isNeonAvailable())) {
14742 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14743 }
14744 return Register();
14745}
14746
14747Register fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14748 switch (VT.SimpleTy) {
14749 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14750 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14751 default: return Register();
14752 }
14753}
14754
14755// FastEmit functions for AArch64ISD::UQSHL_I.
14756
14757Register fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14758 if (RetVT.SimpleTy != MVT::v8i8)
14759 return Register();
14760 if ((Subtarget->isNeonAvailable())) {
14761 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14762 }
14763 return Register();
14764}
14765
14766Register fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14767 if (RetVT.SimpleTy != MVT::v16i8)
14768 return Register();
14769 if ((Subtarget->isNeonAvailable())) {
14770 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14771 }
14772 return Register();
14773}
14774
14775Register fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14776 switch (VT.SimpleTy) {
14777 case MVT::v8i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14778 case MVT::v16i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14779 default: return Register();
14780 }
14781}
14782
14783// FastEmit functions for AArch64ISD::VSHL.
14784
14785Register fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14786 if (RetVT.SimpleTy != MVT::v8i8)
14787 return Register();
14788 if ((Subtarget->isNeonAvailable())) {
14789 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14790 }
14791 return Register();
14792}
14793
14794Register fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, Register Op0, uint64_t imm1) {
14795 if (RetVT.SimpleTy != MVT::v16i8)
14796 return Register();
14797 if ((Subtarget->isNeonAvailable())) {
14798 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14799 }
14800 return Register();
14801}
14802
14803Register fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14804 switch (VT.SimpleTy) {
14805 case MVT::v8i8: return fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14806 case MVT::v16i8: return fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
14807 default: return Register();
14808 }
14809}
14810
14811// Top-level FastEmit function.
14812
14813Register fastEmit_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14814 switch (Opcode) {
14815 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
14816 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
14817 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
14818 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
14819 default: return Register();
14820 }
14821}
14822
14823// FastEmit functions for AArch64ISD::SQSHLU_I.
14824
14825Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14826 if (RetVT.SimpleTy != MVT::v4i16)
14827 return Register();
14828 if ((Subtarget->isNeonAvailable())) {
14829 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14830 }
14831 return Register();
14832}
14833
14834Register fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14835 if (RetVT.SimpleTy != MVT::v8i16)
14836 return Register();
14837 if ((Subtarget->isNeonAvailable())) {
14838 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14839 }
14840 return Register();
14841}
14842
14843Register fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14844 switch (VT.SimpleTy) {
14845 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14846 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14847 default: return Register();
14848 }
14849}
14850
14851// FastEmit functions for AArch64ISD::SQSHL_I.
14852
14853Register fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14854 if (RetVT.SimpleTy != MVT::v4i16)
14855 return Register();
14856 if ((Subtarget->isNeonAvailable())) {
14857 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14858 }
14859 return Register();
14860}
14861
14862Register fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14863 if (RetVT.SimpleTy != MVT::v8i16)
14864 return Register();
14865 if ((Subtarget->isNeonAvailable())) {
14866 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14867 }
14868 return Register();
14869}
14870
14871Register fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14872 switch (VT.SimpleTy) {
14873 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14874 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14875 default: return Register();
14876 }
14877}
14878
14879// FastEmit functions for AArch64ISD::UQSHL_I.
14880
14881Register fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14882 if (RetVT.SimpleTy != MVT::v4i16)
14883 return Register();
14884 if ((Subtarget->isNeonAvailable())) {
14885 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14886 }
14887 return Register();
14888}
14889
14890Register fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14891 if (RetVT.SimpleTy != MVT::v8i16)
14892 return Register();
14893 if ((Subtarget->isNeonAvailable())) {
14894 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14895 }
14896 return Register();
14897}
14898
14899Register fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14900 switch (VT.SimpleTy) {
14901 case MVT::v4i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14902 case MVT::v8i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14903 default: return Register();
14904 }
14905}
14906
14907// FastEmit functions for AArch64ISD::VSHL.
14908
14909Register fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14910 if (RetVT.SimpleTy != MVT::v4i16)
14911 return Register();
14912 if ((Subtarget->isNeonAvailable())) {
14913 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14914 }
14915 return Register();
14916}
14917
14918Register fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, Register Op0, uint64_t imm1) {
14919 if (RetVT.SimpleTy != MVT::v8i16)
14920 return Register();
14921 if ((Subtarget->isNeonAvailable())) {
14922 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14923 }
14924 return Register();
14925}
14926
14927Register fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14928 switch (VT.SimpleTy) {
14929 case MVT::v4i16: return fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14930 case MVT::v8i16: return fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
14931 default: return Register();
14932 }
14933}
14934
14935// Top-level FastEmit function.
14936
14937Register fastEmit_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
14938 switch (Opcode) {
14939 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
14940 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
14941 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
14942 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
14943 default: return Register();
14944 }
14945}
14946
14947// FastEmit functions for AArch64ISD::SRSHR_I.
14948
14949Register fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
14950 if (RetVT.SimpleTy != MVT::v8i8)
14951 return Register();
14952 if ((Subtarget->isNeonAvailable())) {
14953 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14954 }
14955 return Register();
14956}
14957
14958Register fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
14959 if (RetVT.SimpleTy != MVT::v16i8)
14960 return Register();
14961 if ((Subtarget->isNeonAvailable())) {
14962 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14963 }
14964 return Register();
14965}
14966
14967Register fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14968 switch (VT.SimpleTy) {
14969 case MVT::v8i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
14970 case MVT::v16i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
14971 default: return Register();
14972 }
14973}
14974
14975// FastEmit functions for AArch64ISD::URSHR_I.
14976
14977Register fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
14978 if (RetVT.SimpleTy != MVT::v8i8)
14979 return Register();
14980 if ((Subtarget->isNeonAvailable())) {
14981 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14982 }
14983 return Register();
14984}
14985
14986Register fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
14987 if (RetVT.SimpleTy != MVT::v16i8)
14988 return Register();
14989 if ((Subtarget->isNeonAvailable())) {
14990 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14991 }
14992 return Register();
14993}
14994
14995Register fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
14996 switch (VT.SimpleTy) {
14997 case MVT::v8i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
14998 case MVT::v16i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
14999 default: return Register();
15000 }
15001}
15002
15003// FastEmit functions for AArch64ISD::VASHR.
15004
15005Register fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
15006 if (RetVT.SimpleTy != MVT::v8i8)
15007 return Register();
15008 if ((Subtarget->isNeonAvailable())) {
15009 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15010 }
15011 return Register();
15012}
15013
15014Register fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
15015 if (RetVT.SimpleTy != MVT::v16i8)
15016 return Register();
15017 if ((Subtarget->isNeonAvailable())) {
15018 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15019 }
15020 return Register();
15021}
15022
15023Register fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15024 switch (VT.SimpleTy) {
15025 case MVT::v8i8: return fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15026 case MVT::v16i8: return fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15027 default: return Register();
15028 }
15029}
15030
15031// FastEmit functions for AArch64ISD::VLSHR.
15032
15033Register fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
15034 if (RetVT.SimpleTy != MVT::v8i8)
15035 return Register();
15036 if ((Subtarget->isNeonAvailable())) {
15037 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15038 }
15039 return Register();
15040}
15041
15042Register fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, Register Op0, uint64_t imm1) {
15043 if (RetVT.SimpleTy != MVT::v16i8)
15044 return Register();
15045 if ((Subtarget->isNeonAvailable())) {
15046 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15047 }
15048 return Register();
15049}
15050
15051Register fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15052 switch (VT.SimpleTy) {
15053 case MVT::v8i8: return fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15054 case MVT::v16i8: return fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15055 default: return Register();
15056 }
15057}
15058
15059// Top-level FastEmit function.
15060
15061Register fastEmit_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15062 switch (Opcode) {
15063 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15064 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15065 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15066 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15067 default: return Register();
15068 }
15069}
15070
15071// FastEmit functions for AArch64ISD::SRSHR_I.
15072
15073Register fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15074 if (RetVT.SimpleTy != MVT::v4i16)
15075 return Register();
15076 if ((Subtarget->isNeonAvailable())) {
15077 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15078 }
15079 return Register();
15080}
15081
15082Register fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15083 if (RetVT.SimpleTy != MVT::v8i16)
15084 return Register();
15085 if ((Subtarget->isNeonAvailable())) {
15086 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15087 }
15088 return Register();
15089}
15090
15091Register fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15092 switch (VT.SimpleTy) {
15093 case MVT::v4i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15094 case MVT::v8i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15095 default: return Register();
15096 }
15097}
15098
15099// FastEmit functions for AArch64ISD::URSHR_I.
15100
15101Register fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15102 if (RetVT.SimpleTy != MVT::v4i16)
15103 return Register();
15104 if ((Subtarget->isNeonAvailable())) {
15105 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15106 }
15107 return Register();
15108}
15109
15110Register fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15111 if (RetVT.SimpleTy != MVT::v8i16)
15112 return Register();
15113 if ((Subtarget->isNeonAvailable())) {
15114 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15115 }
15116 return Register();
15117}
15118
15119Register fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15120 switch (VT.SimpleTy) {
15121 case MVT::v4i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15122 case MVT::v8i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15123 default: return Register();
15124 }
15125}
15126
15127// FastEmit functions for AArch64ISD::VASHR.
15128
15129Register fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15130 if (RetVT.SimpleTy != MVT::v4i16)
15131 return Register();
15132 if ((Subtarget->isNeonAvailable())) {
15133 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15134 }
15135 return Register();
15136}
15137
15138Register fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15139 if (RetVT.SimpleTy != MVT::v8i16)
15140 return Register();
15141 if ((Subtarget->isNeonAvailable())) {
15142 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15143 }
15144 return Register();
15145}
15146
15147Register fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15148 switch (VT.SimpleTy) {
15149 case MVT::v4i16: return fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15150 case MVT::v8i16: return fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15151 default: return Register();
15152 }
15153}
15154
15155// FastEmit functions for AArch64ISD::VLSHR.
15156
15157Register fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15158 if (RetVT.SimpleTy != MVT::v4i16)
15159 return Register();
15160 if ((Subtarget->isNeonAvailable())) {
15161 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15162 }
15163 return Register();
15164}
15165
15166Register fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, Register Op0, uint64_t imm1) {
15167 if (RetVT.SimpleTy != MVT::v8i16)
15168 return Register();
15169 if ((Subtarget->isNeonAvailable())) {
15170 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15171 }
15172 return Register();
15173}
15174
15175Register fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15176 switch (VT.SimpleTy) {
15177 case MVT::v4i16: return fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15178 case MVT::v8i16: return fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15179 default: return Register();
15180 }
15181}
15182
15183// Top-level FastEmit function.
15184
15185Register fastEmit_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15186 switch (Opcode) {
15187 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15188 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15189 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15190 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15191 default: return Register();
15192 }
15193}
15194
15195// FastEmit functions for AArch64ISD::SRSHR_I.
15196
15197Register fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15198 if (RetVT.SimpleTy != MVT::v2i32)
15199 return Register();
15200 if ((Subtarget->isNeonAvailable())) {
15201 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15202 }
15203 return Register();
15204}
15205
15206Register fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15207 if (RetVT.SimpleTy != MVT::v4i32)
15208 return Register();
15209 if ((Subtarget->isNeonAvailable())) {
15210 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15211 }
15212 return Register();
15213}
15214
15215Register fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15216 switch (VT.SimpleTy) {
15217 case MVT::v2i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15218 case MVT::v4i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15219 default: return Register();
15220 }
15221}
15222
15223// FastEmit functions for AArch64ISD::URSHR_I.
15224
15225Register fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15226 if (RetVT.SimpleTy != MVT::v2i32)
15227 return Register();
15228 if ((Subtarget->isNeonAvailable())) {
15229 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15230 }
15231 return Register();
15232}
15233
15234Register fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15235 if (RetVT.SimpleTy != MVT::v4i32)
15236 return Register();
15237 if ((Subtarget->isNeonAvailable())) {
15238 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15239 }
15240 return Register();
15241}
15242
15243Register fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15244 switch (VT.SimpleTy) {
15245 case MVT::v2i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15246 case MVT::v4i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15247 default: return Register();
15248 }
15249}
15250
15251// FastEmit functions for AArch64ISD::VASHR.
15252
15253Register fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15254 if (RetVT.SimpleTy != MVT::v2i32)
15255 return Register();
15256 if ((Subtarget->isNeonAvailable())) {
15257 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15258 }
15259 return Register();
15260}
15261
15262Register fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15263 if (RetVT.SimpleTy != MVT::v4i32)
15264 return Register();
15265 if ((Subtarget->isNeonAvailable())) {
15266 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15267 }
15268 return Register();
15269}
15270
15271Register fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15272 switch (VT.SimpleTy) {
15273 case MVT::v2i32: return fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15274 case MVT::v4i32: return fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15275 default: return Register();
15276 }
15277}
15278
15279// FastEmit functions for AArch64ISD::VLSHR.
15280
15281Register fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15282 if (RetVT.SimpleTy != MVT::v2i32)
15283 return Register();
15284 if ((Subtarget->isNeonAvailable())) {
15285 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15286 }
15287 return Register();
15288}
15289
15290Register fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, Register Op0, uint64_t imm1) {
15291 if (RetVT.SimpleTy != MVT::v4i32)
15292 return Register();
15293 if ((Subtarget->isNeonAvailable())) {
15294 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15295 }
15296 return Register();
15297}
15298
15299Register fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15300 switch (VT.SimpleTy) {
15301 case MVT::v2i32: return fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15302 case MVT::v4i32: return fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15303 default: return Register();
15304 }
15305}
15306
15307// Top-level FastEmit function.
15308
15309Register fastEmit_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15310 switch (Opcode) {
15311 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15312 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15313 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15314 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15315 default: return Register();
15316 }
15317}
15318
15319// FastEmit functions for ISD::SMAX.
15320
15321Register fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
15322 if (RetVT.SimpleTy != MVT::i32)
15323 return Register();
15324 if ((Subtarget->hasCSSC())) {
15325 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15326 }
15327 return Register();
15328}
15329
15330Register fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15331 switch (VT.SimpleTy) {
15332 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
15333 default: return Register();
15334 }
15335}
15336
15337// FastEmit functions for ISD::SMIN.
15338
15339Register fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
15340 if (RetVT.SimpleTy != MVT::i32)
15341 return Register();
15342 if ((Subtarget->hasCSSC())) {
15343 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15344 }
15345 return Register();
15346}
15347
15348Register fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15349 switch (VT.SimpleTy) {
15350 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
15351 default: return Register();
15352 }
15353}
15354
15355// Top-level FastEmit function.
15356
15357Register fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15358 switch (Opcode) {
15359 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
15360 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
15361 default: return Register();
15362 }
15363}
15364
15365// FastEmit functions for ISD::SMAX.
15366
15367Register fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
15368 if (RetVT.SimpleTy != MVT::i64)
15369 return Register();
15370 if ((Subtarget->hasCSSC())) {
15371 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15372 }
15373 return Register();
15374}
15375
15376Register fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15377 switch (VT.SimpleTy) {
15378 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
15379 default: return Register();
15380 }
15381}
15382
15383// FastEmit functions for ISD::SMIN.
15384
15385Register fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
15386 if (RetVT.SimpleTy != MVT::i64)
15387 return Register();
15388 if ((Subtarget->hasCSSC())) {
15389 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15390 }
15391 return Register();
15392}
15393
15394Register fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15395 switch (VT.SimpleTy) {
15396 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
15397 default: return Register();
15398 }
15399}
15400
15401// Top-level FastEmit function.
15402
15403Register fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15404 switch (Opcode) {
15405 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
15406 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
15407 default: return Register();
15408 }
15409}
15410
15411// FastEmit functions for ISD::UMAX.
15412
15413Register fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
15414 if (RetVT.SimpleTy != MVT::i32)
15415 return Register();
15416 if ((Subtarget->hasCSSC())) {
15417 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15418 }
15419 return Register();
15420}
15421
15422Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15423 switch (VT.SimpleTy) {
15424 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
15425 default: return Register();
15426 }
15427}
15428
15429// FastEmit functions for ISD::UMIN.
15430
15431Register fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, Register Op0, uint64_t imm1) {
15432 if (RetVT.SimpleTy != MVT::i32)
15433 return Register();
15434 if ((Subtarget->hasCSSC())) {
15435 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15436 }
15437 return Register();
15438}
15439
15440Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15441 switch (VT.SimpleTy) {
15442 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
15443 default: return Register();
15444 }
15445}
15446
15447// Top-level FastEmit function.
15448
15449Register fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15450 switch (Opcode) {
15451 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
15452 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
15453 default: return Register();
15454 }
15455}
15456
15457// FastEmit functions for ISD::UMAX.
15458
15459Register fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
15460 if (RetVT.SimpleTy != MVT::i64)
15461 return Register();
15462 if ((Subtarget->hasCSSC())) {
15463 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15464 }
15465 return Register();
15466}
15467
15468Register fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15469 switch (VT.SimpleTy) {
15470 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
15471 default: return Register();
15472 }
15473}
15474
15475// FastEmit functions for ISD::UMIN.
15476
15477Register fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, Register Op0, uint64_t imm1) {
15478 if (RetVT.SimpleTy != MVT::i64)
15479 return Register();
15480 if ((Subtarget->hasCSSC())) {
15481 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15482 }
15483 return Register();
15484}
15485
15486Register fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, Register Op0, uint64_t imm1) {
15487 switch (VT.SimpleTy) {
15488 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
15489 default: return Register();
15490 }
15491}
15492
15493// Top-level FastEmit function.
15494
15495Register fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, Register Op0, uint64_t imm1) {
15496 switch (Opcode) {
15497 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
15498 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
15499 default: return Register();
15500 }
15501}
15502
15503// FastEmit functions for ISD::Constant.
15504
15505Register fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
15506 if (RetVT.SimpleTy != MVT::i32)
15507 return Register();
15508 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
15509}
15510
15511Register fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
15512 if (RetVT.SimpleTy != MVT::i64)
15513 return Register();
15514 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
15515}
15516
15517Register fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
15518 switch (VT.SimpleTy) {
15519 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
15520 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
15521 default: return Register();
15522 }
15523}
15524
15525// Top-level FastEmit function.
15526
15527Register fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
15528 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
15529 if (Register Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
15530 return Reg;
15531
15532 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
15533 if (Register Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
15534 return Reg;
15535
15536 switch (Opcode) {
15537 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
15538 default: return Register();
15539 }
15540}
15541
15542// FastEmit functions for AArch64ISD::FMOV.
15543
15544Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
15545 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
15546 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15547 }
15548 return Register();
15549}
15550
15551Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
15552 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
15553 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15554 }
15555 return Register();
15556}
15557
15558Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
15559 if ((Subtarget->isNeonAvailable())) {
15560 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15561 }
15562 return Register();
15563}
15564
15565Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
15566 if ((Subtarget->isNeonAvailable())) {
15567 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15568 }
15569 return Register();
15570}
15571
15572Register fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
15573 if ((Subtarget->isNeonAvailable())) {
15574 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15575 }
15576 return Register();
15577}
15578
15579Register fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15580switch (RetVT.SimpleTy) {
15581 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
15582 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
15583 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
15584 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
15585 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
15586 default: return Register();
15587}
15588}
15589
15590Register fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15591 switch (VT.SimpleTy) {
15592 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15593 default: return Register();
15594 }
15595}
15596
15597// FastEmit functions for AArch64ISD::MOVI.
15598
15599Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
15600 if ((Subtarget->isNeonAvailable())) {
15601 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15602 }
15603 return Register();
15604}
15605
15606Register fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
15607 if ((Subtarget->isNeonAvailable())) {
15608 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15609 }
15610 return Register();
15611}
15612
15613Register fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15614switch (RetVT.SimpleTy) {
15615 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
15616 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
15617 default: return Register();
15618}
15619}
15620
15621Register fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15622 switch (VT.SimpleTy) {
15623 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15624 default: return Register();
15625 }
15626}
15627
15628// FastEmit functions for AArch64ISD::MOVIedit.
15629
15630Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
15631 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
15632}
15633
15634Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
15635 if ((Subtarget->isNeonAvailable())) {
15636 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15637 }
15638 return Register();
15639}
15640
15641Register fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15642switch (RetVT.SimpleTy) {
15643 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
15644 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
15645 default: return Register();
15646}
15647}
15648
15649Register fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15650 switch (VT.SimpleTy) {
15651 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15652 default: return Register();
15653 }
15654}
15655
15656// Top-level FastEmit function.
15657
15658Register fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15659 switch (Opcode) {
15660 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
15661 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
15662 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
15663 default: return Register();
15664 }
15665}
15666
15667// FastEmit functions for AArch64ISD::RDSVL.
15668
15669Register fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
15670 if (RetVT.SimpleTy != MVT::i64)
15671 return Register();
15672 if ((Subtarget->hasSME())) {
15673 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
15674 }
15675 return Register();
15676}
15677
15678Register fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
15679 switch (VT.SimpleTy) {
15680 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
15681 default: return Register();
15682 }
15683}
15684
15685// Top-level FastEmit function.
15686
15687Register fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15688 switch (Opcode) {
15689 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
15690 default: return Register();
15691 }
15692}
15693
15694