1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the AArch64 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_imm0_31(int64_t Imm) {
12
13 return ((uint64_t)Imm) < 32;
14
15}
16static bool Predicate_imm0_63(int64_t Imm) {
17
18 return ((uint64_t)Imm) < 64;
19
20}
21static bool Predicate_imm32_0_31(int64_t Imm) {
22
23 return ((uint64_t)Imm) < 32;
24
25}
26static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
27
28 return (((uint32_t)Imm) < 32);
29
30}
31static bool Predicate_tbz_imm32_63(int64_t Imm) {
32
33 return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
34
35}
36static bool Predicate_VectorIndexD(int64_t Imm) {
37 return ((uint64_t)Imm) < 2;
38}
39static bool Predicate_VectorIndexS(int64_t Imm) {
40 return ((uint64_t)Imm) < 4;
41}
42static bool Predicate_VectorIndexH(int64_t Imm) {
43 return ((uint64_t)Imm) < 8;
44}
45static bool Predicate_VectorIndexB(int64_t Imm) {
46 return ((uint64_t)Imm) < 16;
47}
48static bool Predicate_VectorIndex0(int64_t Imm) {
49 return ((uint64_t)Imm) == 0;
50}
51static bool Predicate_imm0_255(int64_t Imm) {
52
53 return ((uint32_t)Imm) < 256;
54
55}
56static bool Predicate_vecshiftL64(int64_t Imm) {
57
58 return (((uint32_t)Imm) < 64);
59
60}
61static bool Predicate_vecshiftL32(int64_t Imm) {
62
63 return (((uint32_t)Imm) < 32);
64
65}
66static bool Predicate_vecshiftR64(int64_t Imm) {
67
68 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 65);
69
70}
71static bool Predicate_vecshiftL8(int64_t Imm) {
72
73 return (((uint32_t)Imm) < 8);
74
75}
76static bool Predicate_vecshiftL16(int64_t Imm) {
77
78 return (((uint32_t)Imm) < 16);
79
80}
81static bool Predicate_vecshiftR8(int64_t Imm) {
82
83 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 9);
84
85}
86static bool Predicate_vecshiftR16(int64_t Imm) {
87
88 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
89
90}
91static bool Predicate_vecshiftR32(int64_t Imm) {
92
93 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 33);
94
95}
96static bool Predicate_simm8_32b(int64_t Imm) {
97 return Imm >= -128 && Imm < 128;
98}
99static bool Predicate_simm8_64b(int64_t Imm) {
100 return Imm >= -128 && Imm < 128;
101}
102static bool Predicate_uimm8_32b(int64_t Imm) {
103 return Imm >= 0 && Imm < 256;
104}
105static bool Predicate_uimm8_64b(int64_t Imm) {
106 return Imm >= 0 && Imm < 256;
107}
108static bool Predicate_simm6_32b(int64_t Imm) {
109 return Imm >= -32 && Imm < 32;
110}
111
112
113// FastEmit functions for AArch64ISD::THREAD_POINTER.
114
115unsigned fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
116 if (RetVT.SimpleTy != MVT::i64)
117 return 0;
118 return fastEmitInst_(MachineInstOpcode: AArch64::MOVbaseTLS, RC: &AArch64::GPR64RegClass);
119}
120
121unsigned fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
122 switch (VT.SimpleTy) {
123 case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
124 default: return 0;
125 }
126}
127
128// Top-level FastEmit function.
129
130unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
131 switch (Opcode) {
132 case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
133 default: return 0;
134 }
135}
136
137// FastEmit functions for AArch64ISD::ALLOCATE_ZA_BUFFER.
138
139unsigned fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(MVT RetVT, unsigned Op0) {
140 if (RetVT.SimpleTy != MVT::i64)
141 return 0;
142 return fastEmitInst_r(MachineInstOpcode: AArch64::AllocateZABuffer, RC: &AArch64::GPR64spRegClass, Op0);
143}
144
145unsigned fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(MVT VT, MVT RetVT, unsigned Op0) {
146 switch (VT.SimpleTy) {
147 case MVT::i64: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_MVT_i64_r(RetVT, Op0);
148 default: return 0;
149 }
150}
151
152// FastEmit functions for AArch64ISD::CALL.
153
154unsigned fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0) {
155 if (RetVT.SimpleTy != MVT::isVoid)
156 return 0;
157 if (( MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
158 return fastEmitInst_r(MachineInstOpcode: AArch64::BLRNoIP, RC: &AArch64::GPR64noipRegClass, Op0);
159 }
160 if (( !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() )) {
161 return fastEmitInst_r(MachineInstOpcode: AArch64::BLR, RC: &AArch64::GPR64RegClass, Op0);
162 }
163 return 0;
164}
165
166unsigned fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
167 switch (VT.SimpleTy) {
168 case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0);
169 default: return 0;
170 }
171}
172
173// FastEmit functions for AArch64ISD::CMEQz.
174
175unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
176 if (RetVT.SimpleTy != MVT::v8i8)
177 return 0;
178 if ((Subtarget->isNeonAvailable())) {
179 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv8i8rz, RC: &AArch64::FPR64RegClass, Op0);
180 }
181 return 0;
182}
183
184unsigned fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
185 if (RetVT.SimpleTy != MVT::v16i8)
186 return 0;
187 if ((Subtarget->isNeonAvailable())) {
188 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv16i8rz, RC: &AArch64::FPR128RegClass, Op0);
189 }
190 return 0;
191}
192
193unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
194 if (RetVT.SimpleTy != MVT::v4i16)
195 return 0;
196 if ((Subtarget->isNeonAvailable())) {
197 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
198 }
199 return 0;
200}
201
202unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
203 if (RetVT.SimpleTy != MVT::v8i16)
204 return 0;
205 if ((Subtarget->isNeonAvailable())) {
206 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
207 }
208 return 0;
209}
210
211unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
212 if (RetVT.SimpleTy != MVT::v2i32)
213 return 0;
214 if ((Subtarget->isNeonAvailable())) {
215 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
216 }
217 return 0;
218}
219
220unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
221 if (RetVT.SimpleTy != MVT::v4i32)
222 return 0;
223 if ((Subtarget->isNeonAvailable())) {
224 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
225 }
226 return 0;
227}
228
229unsigned fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
230 if (RetVT.SimpleTy != MVT::v1i64)
231 return 0;
232 if ((Subtarget->isNeonAvailable())) {
233 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
234 }
235 return 0;
236}
237
238unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
239 if (RetVT.SimpleTy != MVT::v2i64)
240 return 0;
241 if ((Subtarget->isNeonAvailable())) {
242 return fastEmitInst_r(MachineInstOpcode: AArch64::CMEQv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
243 }
244 return 0;
245}
246
247unsigned fastEmit_AArch64ISD_CMEQz_r(MVT VT, MVT RetVT, unsigned Op0) {
248 switch (VT.SimpleTy) {
249 case MVT::v8i8: return fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(RetVT, Op0);
250 case MVT::v16i8: return fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(RetVT, Op0);
251 case MVT::v4i16: return fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(RetVT, Op0);
252 case MVT::v8i16: return fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(RetVT, Op0);
253 case MVT::v2i32: return fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(RetVT, Op0);
254 case MVT::v4i32: return fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(RetVT, Op0);
255 case MVT::v1i64: return fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(RetVT, Op0);
256 case MVT::v2i64: return fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(RetVT, Op0);
257 default: return 0;
258 }
259}
260
261// FastEmit functions for AArch64ISD::CMGEz.
262
263unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
264 if (RetVT.SimpleTy != MVT::v8i8)
265 return 0;
266 if ((Subtarget->isNeonAvailable())) {
267 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv8i8rz, RC: &AArch64::FPR64RegClass, Op0);
268 }
269 return 0;
270}
271
272unsigned fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
273 if (RetVT.SimpleTy != MVT::v16i8)
274 return 0;
275 if ((Subtarget->isNeonAvailable())) {
276 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv16i8rz, RC: &AArch64::FPR128RegClass, Op0);
277 }
278 return 0;
279}
280
281unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
282 if (RetVT.SimpleTy != MVT::v4i16)
283 return 0;
284 if ((Subtarget->isNeonAvailable())) {
285 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
286 }
287 return 0;
288}
289
290unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
291 if (RetVT.SimpleTy != MVT::v8i16)
292 return 0;
293 if ((Subtarget->isNeonAvailable())) {
294 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
295 }
296 return 0;
297}
298
299unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
300 if (RetVT.SimpleTy != MVT::v2i32)
301 return 0;
302 if ((Subtarget->isNeonAvailable())) {
303 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
304 }
305 return 0;
306}
307
308unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
309 if (RetVT.SimpleTy != MVT::v4i32)
310 return 0;
311 if ((Subtarget->isNeonAvailable())) {
312 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
313 }
314 return 0;
315}
316
317unsigned fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
318 if (RetVT.SimpleTy != MVT::v1i64)
319 return 0;
320 if ((Subtarget->isNeonAvailable())) {
321 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
322 }
323 return 0;
324}
325
326unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
327 if (RetVT.SimpleTy != MVT::v2i64)
328 return 0;
329 if ((Subtarget->isNeonAvailable())) {
330 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGEv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
331 }
332 return 0;
333}
334
335unsigned fastEmit_AArch64ISD_CMGEz_r(MVT VT, MVT RetVT, unsigned Op0) {
336 switch (VT.SimpleTy) {
337 case MVT::v8i8: return fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(RetVT, Op0);
338 case MVT::v16i8: return fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(RetVT, Op0);
339 case MVT::v4i16: return fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(RetVT, Op0);
340 case MVT::v8i16: return fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(RetVT, Op0);
341 case MVT::v2i32: return fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(RetVT, Op0);
342 case MVT::v4i32: return fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(RetVT, Op0);
343 case MVT::v1i64: return fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(RetVT, Op0);
344 case MVT::v2i64: return fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(RetVT, Op0);
345 default: return 0;
346 }
347}
348
349// FastEmit functions for AArch64ISD::CMGTz.
350
351unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
352 if (RetVT.SimpleTy != MVT::v8i8)
353 return 0;
354 if ((Subtarget->isNeonAvailable())) {
355 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv8i8rz, RC: &AArch64::FPR64RegClass, Op0);
356 }
357 return 0;
358}
359
360unsigned fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
361 if (RetVT.SimpleTy != MVT::v16i8)
362 return 0;
363 if ((Subtarget->isNeonAvailable())) {
364 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv16i8rz, RC: &AArch64::FPR128RegClass, Op0);
365 }
366 return 0;
367}
368
369unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
370 if (RetVT.SimpleTy != MVT::v4i16)
371 return 0;
372 if ((Subtarget->isNeonAvailable())) {
373 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
374 }
375 return 0;
376}
377
378unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
379 if (RetVT.SimpleTy != MVT::v8i16)
380 return 0;
381 if ((Subtarget->isNeonAvailable())) {
382 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
383 }
384 return 0;
385}
386
387unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
388 if (RetVT.SimpleTy != MVT::v2i32)
389 return 0;
390 if ((Subtarget->isNeonAvailable())) {
391 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
392 }
393 return 0;
394}
395
396unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
397 if (RetVT.SimpleTy != MVT::v4i32)
398 return 0;
399 if ((Subtarget->isNeonAvailable())) {
400 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
401 }
402 return 0;
403}
404
405unsigned fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
406 if (RetVT.SimpleTy != MVT::v1i64)
407 return 0;
408 if ((Subtarget->isNeonAvailable())) {
409 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
410 }
411 return 0;
412}
413
414unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
415 if (RetVT.SimpleTy != MVT::v2i64)
416 return 0;
417 if ((Subtarget->isNeonAvailable())) {
418 return fastEmitInst_r(MachineInstOpcode: AArch64::CMGTv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
419 }
420 return 0;
421}
422
423unsigned fastEmit_AArch64ISD_CMGTz_r(MVT VT, MVT RetVT, unsigned Op0) {
424 switch (VT.SimpleTy) {
425 case MVT::v8i8: return fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(RetVT, Op0);
426 case MVT::v16i8: return fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(RetVT, Op0);
427 case MVT::v4i16: return fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(RetVT, Op0);
428 case MVT::v8i16: return fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(RetVT, Op0);
429 case MVT::v2i32: return fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(RetVT, Op0);
430 case MVT::v4i32: return fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(RetVT, Op0);
431 case MVT::v1i64: return fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(RetVT, Op0);
432 case MVT::v2i64: return fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(RetVT, Op0);
433 default: return 0;
434 }
435}
436
437// FastEmit functions for AArch64ISD::CMLEz.
438
439unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
440 if (RetVT.SimpleTy != MVT::v8i8)
441 return 0;
442 if ((Subtarget->isNeonAvailable())) {
443 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv8i8rz, RC: &AArch64::FPR64RegClass, Op0);
444 }
445 return 0;
446}
447
448unsigned fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
449 if (RetVT.SimpleTy != MVT::v16i8)
450 return 0;
451 if ((Subtarget->isNeonAvailable())) {
452 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv16i8rz, RC: &AArch64::FPR128RegClass, Op0);
453 }
454 return 0;
455}
456
457unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
458 if (RetVT.SimpleTy != MVT::v4i16)
459 return 0;
460 if ((Subtarget->isNeonAvailable())) {
461 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
462 }
463 return 0;
464}
465
466unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
467 if (RetVT.SimpleTy != MVT::v8i16)
468 return 0;
469 if ((Subtarget->isNeonAvailable())) {
470 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
471 }
472 return 0;
473}
474
475unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
476 if (RetVT.SimpleTy != MVT::v2i32)
477 return 0;
478 if ((Subtarget->isNeonAvailable())) {
479 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
480 }
481 return 0;
482}
483
484unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
485 if (RetVT.SimpleTy != MVT::v4i32)
486 return 0;
487 if ((Subtarget->isNeonAvailable())) {
488 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
489 }
490 return 0;
491}
492
493unsigned fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
494 if (RetVT.SimpleTy != MVT::v1i64)
495 return 0;
496 if ((Subtarget->isNeonAvailable())) {
497 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
498 }
499 return 0;
500}
501
502unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
503 if (RetVT.SimpleTy != MVT::v2i64)
504 return 0;
505 if ((Subtarget->isNeonAvailable())) {
506 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLEv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
507 }
508 return 0;
509}
510
511unsigned fastEmit_AArch64ISD_CMLEz_r(MVT VT, MVT RetVT, unsigned Op0) {
512 switch (VT.SimpleTy) {
513 case MVT::v8i8: return fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(RetVT, Op0);
514 case MVT::v16i8: return fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(RetVT, Op0);
515 case MVT::v4i16: return fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(RetVT, Op0);
516 case MVT::v8i16: return fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(RetVT, Op0);
517 case MVT::v2i32: return fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(RetVT, Op0);
518 case MVT::v4i32: return fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(RetVT, Op0);
519 case MVT::v1i64: return fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(RetVT, Op0);
520 case MVT::v2i64: return fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(RetVT, Op0);
521 default: return 0;
522 }
523}
524
525// FastEmit functions for AArch64ISD::CMLTz.
526
527unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
528 if (RetVT.SimpleTy != MVT::v8i8)
529 return 0;
530 if ((Subtarget->isNeonAvailable())) {
531 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv8i8rz, RC: &AArch64::FPR64RegClass, Op0);
532 }
533 return 0;
534}
535
536unsigned fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
537 if (RetVT.SimpleTy != MVT::v16i8)
538 return 0;
539 if ((Subtarget->isNeonAvailable())) {
540 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv16i8rz, RC: &AArch64::FPR128RegClass, Op0);
541 }
542 return 0;
543}
544
545unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
546 if (RetVT.SimpleTy != MVT::v4i16)
547 return 0;
548 if ((Subtarget->isNeonAvailable())) {
549 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
550 }
551 return 0;
552}
553
554unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
555 if (RetVT.SimpleTy != MVT::v8i16)
556 return 0;
557 if ((Subtarget->isNeonAvailable())) {
558 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
559 }
560 return 0;
561}
562
563unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
564 if (RetVT.SimpleTy != MVT::v2i32)
565 return 0;
566 if ((Subtarget->isNeonAvailable())) {
567 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
568 }
569 return 0;
570}
571
572unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
573 if (RetVT.SimpleTy != MVT::v4i32)
574 return 0;
575 if ((Subtarget->isNeonAvailable())) {
576 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
577 }
578 return 0;
579}
580
581unsigned fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
582 if (RetVT.SimpleTy != MVT::v1i64)
583 return 0;
584 if ((Subtarget->isNeonAvailable())) {
585 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
586 }
587 return 0;
588}
589
590unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
591 if (RetVT.SimpleTy != MVT::v2i64)
592 return 0;
593 if ((Subtarget->isNeonAvailable())) {
594 return fastEmitInst_r(MachineInstOpcode: AArch64::CMLTv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
595 }
596 return 0;
597}
598
599unsigned fastEmit_AArch64ISD_CMLTz_r(MVT VT, MVT RetVT, unsigned Op0) {
600 switch (VT.SimpleTy) {
601 case MVT::v8i8: return fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(RetVT, Op0);
602 case MVT::v16i8: return fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(RetVT, Op0);
603 case MVT::v4i16: return fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(RetVT, Op0);
604 case MVT::v8i16: return fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(RetVT, Op0);
605 case MVT::v2i32: return fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(RetVT, Op0);
606 case MVT::v4i32: return fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(RetVT, Op0);
607 case MVT::v1i64: return fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(RetVT, Op0);
608 case MVT::v2i64: return fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(RetVT, Op0);
609 default: return 0;
610 }
611}
612
613// FastEmit functions for AArch64ISD::COALESCER_BARRIER.
614
615unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(MVT RetVT, unsigned Op0) {
616 if (RetVT.SimpleTy != MVT::bf16)
617 return 0;
618 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
619}
620
621unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(MVT RetVT, unsigned Op0) {
622 if (RetVT.SimpleTy != MVT::f16)
623 return 0;
624 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR16, RC: &AArch64::FPR16RegClass, Op0);
625}
626
627unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(MVT RetVT, unsigned Op0) {
628 if (RetVT.SimpleTy != MVT::f32)
629 return 0;
630 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR32, RC: &AArch64::FPR32RegClass, Op0);
631}
632
633unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(MVT RetVT, unsigned Op0) {
634 if (RetVT.SimpleTy != MVT::f64)
635 return 0;
636 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
637}
638
639unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(MVT RetVT, unsigned Op0) {
640 if (RetVT.SimpleTy != MVT::f128)
641 return 0;
642 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
643}
644
645unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
646 if (RetVT.SimpleTy != MVT::v8i8)
647 return 0;
648 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
649}
650
651unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
652 if (RetVT.SimpleTy != MVT::v16i8)
653 return 0;
654 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
655}
656
657unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
658 if (RetVT.SimpleTy != MVT::v4i16)
659 return 0;
660 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
661}
662
663unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
664 if (RetVT.SimpleTy != MVT::v8i16)
665 return 0;
666 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
667}
668
669unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
670 if (RetVT.SimpleTy != MVT::v2i32)
671 return 0;
672 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
673}
674
675unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
676 if (RetVT.SimpleTy != MVT::v4i32)
677 return 0;
678 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
679}
680
681unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
682 if (RetVT.SimpleTy != MVT::v1i64)
683 return 0;
684 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
685}
686
687unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
688 if (RetVT.SimpleTy != MVT::v2i64)
689 return 0;
690 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
691}
692
693unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
694 if (RetVT.SimpleTy != MVT::v4f16)
695 return 0;
696 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
697}
698
699unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
700 if (RetVT.SimpleTy != MVT::v8f16)
701 return 0;
702 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
703}
704
705unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
706 if (RetVT.SimpleTy != MVT::v4bf16)
707 return 0;
708 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
709}
710
711unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
712 if (RetVT.SimpleTy != MVT::v8bf16)
713 return 0;
714 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
715}
716
717unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
718 if (RetVT.SimpleTy != MVT::v2f32)
719 return 0;
720 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
721}
722
723unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
724 if (RetVT.SimpleTy != MVT::v4f32)
725 return 0;
726 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
727}
728
729unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
730 if (RetVT.SimpleTy != MVT::v1f64)
731 return 0;
732 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR64, RC: &AArch64::FPR64RegClass, Op0);
733}
734
735unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
736 if (RetVT.SimpleTy != MVT::v2f64)
737 return 0;
738 return fastEmitInst_r(MachineInstOpcode: AArch64::COALESCER_BARRIER_FPR128, RC: &AArch64::FPR128RegClass, Op0);
739}
740
741unsigned fastEmit_AArch64ISD_COALESCER_BARRIER_r(MVT VT, MVT RetVT, unsigned Op0) {
742 switch (VT.SimpleTy) {
743 case MVT::bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_bf16_r(RetVT, Op0);
744 case MVT::f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f16_r(RetVT, Op0);
745 case MVT::f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f32_r(RetVT, Op0);
746 case MVT::f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f64_r(RetVT, Op0);
747 case MVT::f128: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_f128_r(RetVT, Op0);
748 case MVT::v8i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i8_r(RetVT, Op0);
749 case MVT::v16i8: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v16i8_r(RetVT, Op0);
750 case MVT::v4i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i16_r(RetVT, Op0);
751 case MVT::v8i16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8i16_r(RetVT, Op0);
752 case MVT::v2i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i32_r(RetVT, Op0);
753 case MVT::v4i32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4i32_r(RetVT, Op0);
754 case MVT::v1i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1i64_r(RetVT, Op0);
755 case MVT::v2i64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2i64_r(RetVT, Op0);
756 case MVT::v4f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f16_r(RetVT, Op0);
757 case MVT::v8f16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8f16_r(RetVT, Op0);
758 case MVT::v4bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4bf16_r(RetVT, Op0);
759 case MVT::v8bf16: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v8bf16_r(RetVT, Op0);
760 case MVT::v2f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f32_r(RetVT, Op0);
761 case MVT::v4f32: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v4f32_r(RetVT, Op0);
762 case MVT::v1f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v1f64_r(RetVT, Op0);
763 case MVT::v2f64: return fastEmit_AArch64ISD_COALESCER_BARRIER_MVT_v2f64_r(RetVT, Op0);
764 default: return 0;
765 }
766}
767
768// FastEmit functions for AArch64ISD::DUP.
769
770unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(unsigned Op0) {
771 if ((Subtarget->isNeonAvailable())) {
772 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i8gpr, RC: &AArch64::FPR64RegClass, Op0);
773 }
774 return 0;
775}
776
777unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(unsigned Op0) {
778 if ((Subtarget->isNeonAvailable())) {
779 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv16i8gpr, RC: &AArch64::FPR128RegClass, Op0);
780 }
781 return 0;
782}
783
784unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(unsigned Op0) {
785 if ((Subtarget->isNeonAvailable())) {
786 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i16gpr, RC: &AArch64::FPR64RegClass, Op0);
787 }
788 return 0;
789}
790
791unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(unsigned Op0) {
792 if ((Subtarget->isNeonAvailable())) {
793 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv8i16gpr, RC: &AArch64::FPR128RegClass, Op0);
794 }
795 return 0;
796}
797
798unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(unsigned Op0) {
799 if ((Subtarget->isNeonAvailable())) {
800 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i32gpr, RC: &AArch64::FPR64RegClass, Op0);
801 }
802 return 0;
803}
804
805unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(unsigned Op0) {
806 if ((Subtarget->isNeonAvailable())) {
807 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv4i32gpr, RC: &AArch64::FPR128RegClass, Op0);
808 }
809 return 0;
810}
811
812unsigned fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, unsigned Op0) {
813switch (RetVT.SimpleTy) {
814 case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0);
815 case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0);
816 case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0);
817 case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0);
818 case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0);
819 case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0);
820 default: return 0;
821}
822}
823
824unsigned fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, unsigned Op0) {
825 if (RetVT.SimpleTy != MVT::v2i64)
826 return 0;
827 if ((Subtarget->isNeonAvailable())) {
828 return fastEmitInst_r(MachineInstOpcode: AArch64::DUPv2i64gpr, RC: &AArch64::FPR128RegClass, Op0);
829 }
830 return 0;
831}
832
833unsigned fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, unsigned Op0) {
834 switch (VT.SimpleTy) {
835 case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0);
836 case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0);
837 default: return 0;
838 }
839}
840
841// FastEmit functions for AArch64ISD::FCMEQz.
842
843unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
844 if (RetVT.SimpleTy != MVT::v4i16)
845 return 0;
846 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
847 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMEQv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
848 }
849 return 0;
850}
851
852unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
853 if (RetVT.SimpleTy != MVT::v8i16)
854 return 0;
855 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
856 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMEQv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
857 }
858 return 0;
859}
860
861unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
862 if (RetVT.SimpleTy != MVT::v2i32)
863 return 0;
864 if ((Subtarget->isNeonAvailable())) {
865 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMEQv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
866 }
867 return 0;
868}
869
870unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
871 if (RetVT.SimpleTy != MVT::v4i32)
872 return 0;
873 if ((Subtarget->isNeonAvailable())) {
874 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMEQv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
875 }
876 return 0;
877}
878
879unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
880 if (RetVT.SimpleTy != MVT::v1i64)
881 return 0;
882 if ((Subtarget->isNeonAvailable())) {
883 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMEQv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
884 }
885 return 0;
886}
887
888unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
889 if (RetVT.SimpleTy != MVT::v2i64)
890 return 0;
891 if ((Subtarget->isNeonAvailable())) {
892 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMEQv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
893 }
894 return 0;
895}
896
897unsigned fastEmit_AArch64ISD_FCMEQz_r(MVT VT, MVT RetVT, unsigned Op0) {
898 switch (VT.SimpleTy) {
899 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(RetVT, Op0);
900 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(RetVT, Op0);
901 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(RetVT, Op0);
902 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(RetVT, Op0);
903 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(RetVT, Op0);
904 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(RetVT, Op0);
905 default: return 0;
906 }
907}
908
909// FastEmit functions for AArch64ISD::FCMGEz.
910
911unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
912 if (RetVT.SimpleTy != MVT::v4i16)
913 return 0;
914 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
915 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGEv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
916 }
917 return 0;
918}
919
920unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
921 if (RetVT.SimpleTy != MVT::v8i16)
922 return 0;
923 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
924 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGEv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
925 }
926 return 0;
927}
928
929unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
930 if (RetVT.SimpleTy != MVT::v2i32)
931 return 0;
932 if ((Subtarget->isNeonAvailable())) {
933 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGEv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
934 }
935 return 0;
936}
937
938unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
939 if (RetVT.SimpleTy != MVT::v4i32)
940 return 0;
941 if ((Subtarget->isNeonAvailable())) {
942 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGEv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
943 }
944 return 0;
945}
946
947unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
948 if (RetVT.SimpleTy != MVT::v1i64)
949 return 0;
950 if ((Subtarget->isNeonAvailable())) {
951 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGEv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
952 }
953 return 0;
954}
955
956unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
957 if (RetVT.SimpleTy != MVT::v2i64)
958 return 0;
959 if ((Subtarget->isNeonAvailable())) {
960 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGEv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
961 }
962 return 0;
963}
964
965unsigned fastEmit_AArch64ISD_FCMGEz_r(MVT VT, MVT RetVT, unsigned Op0) {
966 switch (VT.SimpleTy) {
967 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(RetVT, Op0);
968 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(RetVT, Op0);
969 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(RetVT, Op0);
970 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(RetVT, Op0);
971 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(RetVT, Op0);
972 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(RetVT, Op0);
973 default: return 0;
974 }
975}
976
977// FastEmit functions for AArch64ISD::FCMGTz.
978
979unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
980 if (RetVT.SimpleTy != MVT::v4i16)
981 return 0;
982 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
983 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGTv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
984 }
985 return 0;
986}
987
988unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
989 if (RetVT.SimpleTy != MVT::v8i16)
990 return 0;
991 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
992 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGTv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
993 }
994 return 0;
995}
996
997unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
998 if (RetVT.SimpleTy != MVT::v2i32)
999 return 0;
1000 if ((Subtarget->isNeonAvailable())) {
1001 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGTv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
1002 }
1003 return 0;
1004}
1005
1006unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1007 if (RetVT.SimpleTy != MVT::v4i32)
1008 return 0;
1009 if ((Subtarget->isNeonAvailable())) {
1010 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGTv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
1011 }
1012 return 0;
1013}
1014
1015unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
1016 if (RetVT.SimpleTy != MVT::v1i64)
1017 return 0;
1018 if ((Subtarget->isNeonAvailable())) {
1019 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGTv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
1020 }
1021 return 0;
1022}
1023
1024unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1025 if (RetVT.SimpleTy != MVT::v2i64)
1026 return 0;
1027 if ((Subtarget->isNeonAvailable())) {
1028 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMGTv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
1029 }
1030 return 0;
1031}
1032
1033unsigned fastEmit_AArch64ISD_FCMGTz_r(MVT VT, MVT RetVT, unsigned Op0) {
1034 switch (VT.SimpleTy) {
1035 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(RetVT, Op0);
1036 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(RetVT, Op0);
1037 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(RetVT, Op0);
1038 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(RetVT, Op0);
1039 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(RetVT, Op0);
1040 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(RetVT, Op0);
1041 default: return 0;
1042 }
1043}
1044
1045// FastEmit functions for AArch64ISD::FCMLEz.
1046
1047unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
1048 if (RetVT.SimpleTy != MVT::v4i16)
1049 return 0;
1050 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
1051 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLEv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
1052 }
1053 return 0;
1054}
1055
1056unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1057 if (RetVT.SimpleTy != MVT::v8i16)
1058 return 0;
1059 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
1060 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLEv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
1061 }
1062 return 0;
1063}
1064
1065unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
1066 if (RetVT.SimpleTy != MVT::v2i32)
1067 return 0;
1068 if ((Subtarget->isNeonAvailable())) {
1069 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLEv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
1070 }
1071 return 0;
1072}
1073
1074unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1075 if (RetVT.SimpleTy != MVT::v4i32)
1076 return 0;
1077 if ((Subtarget->isNeonAvailable())) {
1078 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLEv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
1079 }
1080 return 0;
1081}
1082
1083unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
1084 if (RetVT.SimpleTy != MVT::v1i64)
1085 return 0;
1086 if ((Subtarget->isNeonAvailable())) {
1087 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLEv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
1088 }
1089 return 0;
1090}
1091
1092unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1093 if (RetVT.SimpleTy != MVT::v2i64)
1094 return 0;
1095 if ((Subtarget->isNeonAvailable())) {
1096 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLEv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
1097 }
1098 return 0;
1099}
1100
1101unsigned fastEmit_AArch64ISD_FCMLEz_r(MVT VT, MVT RetVT, unsigned Op0) {
1102 switch (VT.SimpleTy) {
1103 case MVT::v4f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(RetVT, Op0);
1104 case MVT::v8f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(RetVT, Op0);
1105 case MVT::v2f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(RetVT, Op0);
1106 case MVT::v4f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(RetVT, Op0);
1107 case MVT::v1f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(RetVT, Op0);
1108 case MVT::v2f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(RetVT, Op0);
1109 default: return 0;
1110 }
1111}
1112
1113// FastEmit functions for AArch64ISD::FCMLTz.
1114
1115unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
1116 if (RetVT.SimpleTy != MVT::v4i16)
1117 return 0;
1118 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
1119 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLTv4i16rz, RC: &AArch64::FPR64RegClass, Op0);
1120 }
1121 return 0;
1122}
1123
1124unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1125 if (RetVT.SimpleTy != MVT::v8i16)
1126 return 0;
1127 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
1128 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLTv8i16rz, RC: &AArch64::FPR128RegClass, Op0);
1129 }
1130 return 0;
1131}
1132
1133unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
1134 if (RetVT.SimpleTy != MVT::v2i32)
1135 return 0;
1136 if ((Subtarget->isNeonAvailable())) {
1137 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLTv2i32rz, RC: &AArch64::FPR64RegClass, Op0);
1138 }
1139 return 0;
1140}
1141
1142unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1143 if (RetVT.SimpleTy != MVT::v4i32)
1144 return 0;
1145 if ((Subtarget->isNeonAvailable())) {
1146 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLTv4i32rz, RC: &AArch64::FPR128RegClass, Op0);
1147 }
1148 return 0;
1149}
1150
1151unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
1152 if (RetVT.SimpleTy != MVT::v1i64)
1153 return 0;
1154 if ((Subtarget->isNeonAvailable())) {
1155 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLTv1i64rz, RC: &AArch64::FPR64RegClass, Op0);
1156 }
1157 return 0;
1158}
1159
1160unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1161 if (RetVT.SimpleTy != MVT::v2i64)
1162 return 0;
1163 if ((Subtarget->isNeonAvailable())) {
1164 return fastEmitInst_r(MachineInstOpcode: AArch64::FCMLTv2i64rz, RC: &AArch64::FPR128RegClass, Op0);
1165 }
1166 return 0;
1167}
1168
1169unsigned fastEmit_AArch64ISD_FCMLTz_r(MVT VT, MVT RetVT, unsigned Op0) {
1170 switch (VT.SimpleTy) {
1171 case MVT::v4f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(RetVT, Op0);
1172 case MVT::v8f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(RetVT, Op0);
1173 case MVT::v2f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(RetVT, Op0);
1174 case MVT::v4f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(RetVT, Op0);
1175 case MVT::v1f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(RetVT, Op0);
1176 case MVT::v2f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(RetVT, Op0);
1177 default: return 0;
1178 }
1179}
1180
1181// FastEmit functions for AArch64ISD::FCVTXN.
1182
1183unsigned fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(MVT RetVT, unsigned Op0) {
1184 if (RetVT.SimpleTy != MVT::f32)
1185 return 0;
1186 if ((Subtarget->isNeonAvailable())) {
1187 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv1i64, RC: &AArch64::FPR32RegClass, Op0);
1188 }
1189 return 0;
1190}
1191
1192unsigned fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1193 if (RetVT.SimpleTy != MVT::v2f32)
1194 return 0;
1195 if ((Subtarget->isNeonAvailable())) {
1196 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTXNv2f32, RC: &AArch64::FPR64RegClass, Op0);
1197 }
1198 return 0;
1199}
1200
1201unsigned fastEmit_AArch64ISD_FCVTXN_r(MVT VT, MVT RetVT, unsigned Op0) {
1202 switch (VT.SimpleTy) {
1203 case MVT::f64: return fastEmit_AArch64ISD_FCVTXN_MVT_f64_r(RetVT, Op0);
1204 case MVT::v2f64: return fastEmit_AArch64ISD_FCVTXN_MVT_v2f64_r(RetVT, Op0);
1205 default: return 0;
1206 }
1207}
1208
1209// FastEmit functions for AArch64ISD::FRECPE.
1210
1211unsigned fastEmit_AArch64ISD_FRECPE_MVT_f32_r(MVT RetVT, unsigned Op0) {
1212 if (RetVT.SimpleTy != MVT::f32)
1213 return 0;
1214 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv1i32, RC: &AArch64::FPR32RegClass, Op0);
1215}
1216
1217unsigned fastEmit_AArch64ISD_FRECPE_MVT_f64_r(MVT RetVT, unsigned Op0) {
1218 if (RetVT.SimpleTy != MVT::f64)
1219 return 0;
1220 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv1i64, RC: &AArch64::FPR64RegClass, Op0);
1221}
1222
1223unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
1224 if (RetVT.SimpleTy != MVT::v2f32)
1225 return 0;
1226 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f32, RC: &AArch64::FPR64RegClass, Op0);
1227}
1228
1229unsigned fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1230 if (RetVT.SimpleTy != MVT::v4f32)
1231 return 0;
1232 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv4f32, RC: &AArch64::FPR128RegClass, Op0);
1233}
1234
1235unsigned fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
1236 if (RetVT.SimpleTy != MVT::v1f64)
1237 return 0;
1238 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv1i64, RC: &AArch64::FPR64RegClass, Op0);
1239}
1240
1241unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1242 if (RetVT.SimpleTy != MVT::v2f64)
1243 return 0;
1244 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPEv2f64, RC: &AArch64::FPR128RegClass, Op0);
1245}
1246
1247unsigned fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(MVT RetVT, unsigned Op0) {
1248 if (RetVT.SimpleTy != MVT::nxv8f16)
1249 return 0;
1250 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1251 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1252 }
1253 return 0;
1254}
1255
1256unsigned fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(MVT RetVT, unsigned Op0) {
1257 if (RetVT.SimpleTy != MVT::nxv4f32)
1258 return 0;
1259 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1260 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1261 }
1262 return 0;
1263}
1264
1265unsigned fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(MVT RetVT, unsigned Op0) {
1266 if (RetVT.SimpleTy != MVT::nxv2f64)
1267 return 0;
1268 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1269 return fastEmitInst_r(MachineInstOpcode: AArch64::FRECPE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1270 }
1271 return 0;
1272}
1273
1274unsigned fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, unsigned Op0) {
1275 switch (VT.SimpleTy) {
1276 case MVT::f32: return fastEmit_AArch64ISD_FRECPE_MVT_f32_r(RetVT, Op0);
1277 case MVT::f64: return fastEmit_AArch64ISD_FRECPE_MVT_f64_r(RetVT, Op0);
1278 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0);
1279 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0);
1280 case MVT::v1f64: return fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(RetVT, Op0);
1281 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0);
1282 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPE_MVT_nxv8f16_r(RetVT, Op0);
1283 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPE_MVT_nxv4f32_r(RetVT, Op0);
1284 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPE_MVT_nxv2f64_r(RetVT, Op0);
1285 default: return 0;
1286 }
1287}
1288
1289// FastEmit functions for AArch64ISD::FRSQRTE.
1290
1291unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(MVT RetVT, unsigned Op0) {
1292 if (RetVT.SimpleTy != MVT::f32)
1293 return 0;
1294 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv1i32, RC: &AArch64::FPR32RegClass, Op0);
1295}
1296
1297unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(MVT RetVT, unsigned Op0) {
1298 if (RetVT.SimpleTy != MVT::f64)
1299 return 0;
1300 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv1i64, RC: &AArch64::FPR64RegClass, Op0);
1301}
1302
1303unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
1304 if (RetVT.SimpleTy != MVT::v2f32)
1305 return 0;
1306 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f32, RC: &AArch64::FPR64RegClass, Op0);
1307}
1308
1309unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1310 if (RetVT.SimpleTy != MVT::v4f32)
1311 return 0;
1312 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv4f32, RC: &AArch64::FPR128RegClass, Op0);
1313}
1314
1315unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
1316 if (RetVT.SimpleTy != MVT::v1f64)
1317 return 0;
1318 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv1i64, RC: &AArch64::FPR64RegClass, Op0);
1319}
1320
1321unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
1322 if (RetVT.SimpleTy != MVT::v2f64)
1323 return 0;
1324 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTEv2f64, RC: &AArch64::FPR128RegClass, Op0);
1325}
1326
1327unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(MVT RetVT, unsigned Op0) {
1328 if (RetVT.SimpleTy != MVT::nxv8f16)
1329 return 0;
1330 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1331 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1332 }
1333 return 0;
1334}
1335
1336unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(MVT RetVT, unsigned Op0) {
1337 if (RetVT.SimpleTy != MVT::nxv4f32)
1338 return 0;
1339 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1340 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1341 }
1342 return 0;
1343}
1344
1345unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(MVT RetVT, unsigned Op0) {
1346 if (RetVT.SimpleTy != MVT::nxv2f64)
1347 return 0;
1348 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1349 return fastEmitInst_r(MachineInstOpcode: AArch64::FRSQRTE_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1350 }
1351 return 0;
1352}
1353
1354unsigned fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, unsigned Op0) {
1355 switch (VT.SimpleTy) {
1356 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(RetVT, Op0);
1357 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(RetVT, Op0);
1358 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0);
1359 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0);
1360 case MVT::v1f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(RetVT, Op0);
1361 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0);
1362 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv8f16_r(RetVT, Op0);
1363 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv4f32_r(RetVT, Op0);
1364 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_nxv2f64_r(RetVT, Op0);
1365 default: return 0;
1366 }
1367}
1368
1369// FastEmit functions for AArch64ISD::INIT_TPIDR2OBJ.
1370
1371unsigned fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_r(MVT RetVT, unsigned Op0) {
1372 if (RetVT.SimpleTy != MVT::isVoid)
1373 return 0;
1374 return fastEmitInst_r(MachineInstOpcode: AArch64::InitTPIDR2Obj, RC: &AArch64::GPR64RegClass, Op0);
1375}
1376
1377unsigned fastEmit_AArch64ISD_INIT_TPIDR2OBJ_r(MVT VT, MVT RetVT, unsigned Op0) {
1378 switch (VT.SimpleTy) {
1379 case MVT::i64: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_MVT_i64_r(RetVT, Op0);
1380 default: return 0;
1381 }
1382}
1383
1384// FastEmit functions for AArch64ISD::PROBED_ALLOCA.
1385
1386unsigned fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0) {
1387 if (RetVT.SimpleTy != MVT::isVoid)
1388 return 0;
1389 return fastEmitInst_r(MachineInstOpcode: AArch64::PROBED_STACKALLOC_DYN, RC: &AArch64::GPR64commonRegClass, Op0);
1390}
1391
1392unsigned fastEmit_AArch64ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0) {
1393 switch (VT.SimpleTy) {
1394 case MVT::i64: return fastEmit_AArch64ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0);
1395 default: return 0;
1396 }
1397}
1398
1399// FastEmit functions for AArch64ISD::REV16.
1400
1401unsigned fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
1402 if (RetVT.SimpleTy != MVT::v8i8)
1403 return 0;
1404 if ((Subtarget->isNeonAvailable())) {
1405 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
1406 }
1407 return 0;
1408}
1409
1410unsigned fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1411 if (RetVT.SimpleTy != MVT::v16i8)
1412 return 0;
1413 if ((Subtarget->isNeonAvailable())) {
1414 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
1415 }
1416 return 0;
1417}
1418
1419unsigned fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, unsigned Op0) {
1420 switch (VT.SimpleTy) {
1421 case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0);
1422 case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0);
1423 default: return 0;
1424 }
1425}
1426
1427// FastEmit functions for AArch64ISD::REV32.
1428
1429unsigned fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
1430 if (RetVT.SimpleTy != MVT::v8i8)
1431 return 0;
1432 if ((Subtarget->isNeonAvailable())) {
1433 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
1434 }
1435 return 0;
1436}
1437
1438unsigned fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1439 if (RetVT.SimpleTy != MVT::v16i8)
1440 return 0;
1441 if ((Subtarget->isNeonAvailable())) {
1442 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
1443 }
1444 return 0;
1445}
1446
1447unsigned fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
1448 if (RetVT.SimpleTy != MVT::v4i16)
1449 return 0;
1450 if ((Subtarget->isNeonAvailable())) {
1451 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1452 }
1453 return 0;
1454}
1455
1456unsigned fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1457 if (RetVT.SimpleTy != MVT::v8i16)
1458 return 0;
1459 if ((Subtarget->isNeonAvailable())) {
1460 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1461 }
1462 return 0;
1463}
1464
1465unsigned fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
1466 if (RetVT.SimpleTy != MVT::v4f16)
1467 return 0;
1468 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1469}
1470
1471unsigned fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1472 if (RetVT.SimpleTy != MVT::v8f16)
1473 return 0;
1474 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1475}
1476
1477unsigned fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
1478 if (RetVT.SimpleTy != MVT::v4bf16)
1479 return 0;
1480 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
1481}
1482
1483unsigned fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
1484 if (RetVT.SimpleTy != MVT::v8bf16)
1485 return 0;
1486 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
1487}
1488
1489unsigned fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, unsigned Op0) {
1490 switch (VT.SimpleTy) {
1491 case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0);
1492 case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0);
1493 case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0);
1494 case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0);
1495 case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0);
1496 case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0);
1497 case MVT::v4bf16: return fastEmit_AArch64ISD_REV32_MVT_v4bf16_r(RetVT, Op0);
1498 case MVT::v8bf16: return fastEmit_AArch64ISD_REV32_MVT_v8bf16_r(RetVT, Op0);
1499 default: return 0;
1500 }
1501}
1502
1503// FastEmit functions for AArch64ISD::REV64.
1504
1505unsigned fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
1506 if (RetVT.SimpleTy != MVT::v8i8)
1507 return 0;
1508 if ((Subtarget->isNeonAvailable())) {
1509 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
1510 }
1511 return 0;
1512}
1513
1514unsigned fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1515 if (RetVT.SimpleTy != MVT::v16i8)
1516 return 0;
1517 if ((Subtarget->isNeonAvailable())) {
1518 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
1519 }
1520 return 0;
1521}
1522
1523unsigned fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
1524 if (RetVT.SimpleTy != MVT::v4i16)
1525 return 0;
1526 if ((Subtarget->isNeonAvailable())) {
1527 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1528 }
1529 return 0;
1530}
1531
1532unsigned fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1533 if (RetVT.SimpleTy != MVT::v8i16)
1534 return 0;
1535 if ((Subtarget->isNeonAvailable())) {
1536 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1537 }
1538 return 0;
1539}
1540
1541unsigned fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
1542 if (RetVT.SimpleTy != MVT::v2i32)
1543 return 0;
1544 if ((Subtarget->isNeonAvailable())) {
1545 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1546 }
1547 return 0;
1548}
1549
1550unsigned fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1551 if (RetVT.SimpleTy != MVT::v4i32)
1552 return 0;
1553 if ((Subtarget->isNeonAvailable())) {
1554 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1555 }
1556 return 0;
1557}
1558
1559unsigned fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
1560 if (RetVT.SimpleTy != MVT::v4f16)
1561 return 0;
1562 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1563}
1564
1565unsigned fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
1566 if (RetVT.SimpleTy != MVT::v8f16)
1567 return 0;
1568 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1569}
1570
1571unsigned fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
1572 if (RetVT.SimpleTy != MVT::v4bf16)
1573 return 0;
1574 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
1575}
1576
1577unsigned fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
1578 if (RetVT.SimpleTy != MVT::v8bf16)
1579 return 0;
1580 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
1581}
1582
1583unsigned fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
1584 if (RetVT.SimpleTy != MVT::v2f32)
1585 return 0;
1586 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
1587}
1588
1589unsigned fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
1590 if (RetVT.SimpleTy != MVT::v4f32)
1591 return 0;
1592 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
1593}
1594
1595unsigned fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, unsigned Op0) {
1596 switch (VT.SimpleTy) {
1597 case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0);
1598 case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0);
1599 case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0);
1600 case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0);
1601 case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0);
1602 case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0);
1603 case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0);
1604 case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0);
1605 case MVT::v4bf16: return fastEmit_AArch64ISD_REV64_MVT_v4bf16_r(RetVT, Op0);
1606 case MVT::v8bf16: return fastEmit_AArch64ISD_REV64_MVT_v8bf16_r(RetVT, Op0);
1607 case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0);
1608 case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0);
1609 default: return 0;
1610 }
1611}
1612
1613// FastEmit functions for AArch64ISD::SADDLP.
1614
1615unsigned fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
1616 if (RetVT.SimpleTy != MVT::v4i16)
1617 return 0;
1618 if ((Subtarget->isNeonAvailable())) {
1619 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1620 }
1621 return 0;
1622}
1623
1624unsigned fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1625 if (RetVT.SimpleTy != MVT::v8i16)
1626 return 0;
1627 if ((Subtarget->isNeonAvailable())) {
1628 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1629 }
1630 return 0;
1631}
1632
1633unsigned fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
1634 if (RetVT.SimpleTy != MVT::v2i32)
1635 return 0;
1636 if ((Subtarget->isNeonAvailable())) {
1637 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1638 }
1639 return 0;
1640}
1641
1642unsigned fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1643 if (RetVT.SimpleTy != MVT::v4i32)
1644 return 0;
1645 if ((Subtarget->isNeonAvailable())) {
1646 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1647 }
1648 return 0;
1649}
1650
1651unsigned fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
1652 if (RetVT.SimpleTy != MVT::v1i64)
1653 return 0;
1654 if ((Subtarget->isNeonAvailable())) {
1655 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1656 }
1657 return 0;
1658}
1659
1660unsigned fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1661 if (RetVT.SimpleTy != MVT::v2i64)
1662 return 0;
1663 if ((Subtarget->isNeonAvailable())) {
1664 return fastEmitInst_r(MachineInstOpcode: AArch64::SADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1665 }
1666 return 0;
1667}
1668
1669unsigned fastEmit_AArch64ISD_SADDLP_r(MVT VT, MVT RetVT, unsigned Op0) {
1670 switch (VT.SimpleTy) {
1671 case MVT::v8i8: return fastEmit_AArch64ISD_SADDLP_MVT_v8i8_r(RetVT, Op0);
1672 case MVT::v16i8: return fastEmit_AArch64ISD_SADDLP_MVT_v16i8_r(RetVT, Op0);
1673 case MVT::v4i16: return fastEmit_AArch64ISD_SADDLP_MVT_v4i16_r(RetVT, Op0);
1674 case MVT::v8i16: return fastEmit_AArch64ISD_SADDLP_MVT_v8i16_r(RetVT, Op0);
1675 case MVT::v2i32: return fastEmit_AArch64ISD_SADDLP_MVT_v2i32_r(RetVT, Op0);
1676 case MVT::v4i32: return fastEmit_AArch64ISD_SADDLP_MVT_v4i32_r(RetVT, Op0);
1677 default: return 0;
1678 }
1679}
1680
1681// FastEmit functions for AArch64ISD::SITOF.
1682
1683unsigned fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, unsigned Op0) {
1684 if (RetVT.SimpleTy != MVT::f16)
1685 return 0;
1686 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
1687 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1688 }
1689 return 0;
1690}
1691
1692unsigned fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, unsigned Op0) {
1693 if (RetVT.SimpleTy != MVT::f32)
1694 return 0;
1695 if ((Subtarget->hasNEON())) {
1696 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1697 }
1698 return 0;
1699}
1700
1701unsigned fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, unsigned Op0) {
1702 if (RetVT.SimpleTy != MVT::f64)
1703 return 0;
1704 if ((Subtarget->hasNEON())) {
1705 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1706 }
1707 return 0;
1708}
1709
1710unsigned fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, unsigned Op0) {
1711 switch (VT.SimpleTy) {
1712 case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0);
1713 case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0);
1714 case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0);
1715 default: return 0;
1716 }
1717}
1718
1719// FastEmit functions for AArch64ISD::SUNPKHI.
1720
1721unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) {
1722 if (RetVT.SimpleTy != MVT::nxv8i16)
1723 return 0;
1724 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1725 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1726 }
1727 return 0;
1728}
1729
1730unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) {
1731 if (RetVT.SimpleTy != MVT::nxv4i32)
1732 return 0;
1733 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1734 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1735 }
1736 return 0;
1737}
1738
1739unsigned fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) {
1740 if (RetVT.SimpleTy != MVT::nxv2i64)
1741 return 0;
1742 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1743 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1744 }
1745 return 0;
1746}
1747
1748unsigned fastEmit_AArch64ISD_SUNPKHI_r(MVT VT, MVT RetVT, unsigned Op0) {
1749 switch (VT.SimpleTy) {
1750 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1751 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1752 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1753 default: return 0;
1754 }
1755}
1756
1757// FastEmit functions for AArch64ISD::SUNPKLO.
1758
1759unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) {
1760 if (RetVT.SimpleTy != MVT::nxv8i16)
1761 return 0;
1762 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1763 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1764 }
1765 return 0;
1766}
1767
1768unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) {
1769 if (RetVT.SimpleTy != MVT::nxv4i32)
1770 return 0;
1771 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1772 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1773 }
1774 return 0;
1775}
1776
1777unsigned fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) {
1778 if (RetVT.SimpleTy != MVT::nxv2i64)
1779 return 0;
1780 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1781 return fastEmitInst_r(MachineInstOpcode: AArch64::SUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1782 }
1783 return 0;
1784}
1785
1786unsigned fastEmit_AArch64ISD_SUNPKLO_r(MVT VT, MVT RetVT, unsigned Op0) {
1787 switch (VT.SimpleTy) {
1788 case MVT::nxv16i8: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1789 case MVT::nxv8i16: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1790 case MVT::nxv4i32: return fastEmit_AArch64ISD_SUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1791 default: return 0;
1792 }
1793}
1794
1795// FastEmit functions for AArch64ISD::UADDLP.
1796
1797unsigned fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
1798 if (RetVT.SimpleTy != MVT::v4i16)
1799 return 0;
1800 if ((Subtarget->isNeonAvailable())) {
1801 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i8_v4i16, RC: &AArch64::FPR64RegClass, Op0);
1802 }
1803 return 0;
1804}
1805
1806unsigned fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
1807 if (RetVT.SimpleTy != MVT::v8i16)
1808 return 0;
1809 if ((Subtarget->isNeonAvailable())) {
1810 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv16i8_v8i16, RC: &AArch64::FPR128RegClass, Op0);
1811 }
1812 return 0;
1813}
1814
1815unsigned fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
1816 if (RetVT.SimpleTy != MVT::v2i32)
1817 return 0;
1818 if ((Subtarget->isNeonAvailable())) {
1819 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i16_v2i32, RC: &AArch64::FPR64RegClass, Op0);
1820 }
1821 return 0;
1822}
1823
1824unsigned fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
1825 if (RetVT.SimpleTy != MVT::v4i32)
1826 return 0;
1827 if ((Subtarget->isNeonAvailable())) {
1828 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv8i16_v4i32, RC: &AArch64::FPR128RegClass, Op0);
1829 }
1830 return 0;
1831}
1832
1833unsigned fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
1834 if (RetVT.SimpleTy != MVT::v1i64)
1835 return 0;
1836 if ((Subtarget->isNeonAvailable())) {
1837 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv2i32_v1i64, RC: &AArch64::FPR64RegClass, Op0);
1838 }
1839 return 0;
1840}
1841
1842unsigned fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
1843 if (RetVT.SimpleTy != MVT::v2i64)
1844 return 0;
1845 if ((Subtarget->isNeonAvailable())) {
1846 return fastEmitInst_r(MachineInstOpcode: AArch64::UADDLPv4i32_v2i64, RC: &AArch64::FPR128RegClass, Op0);
1847 }
1848 return 0;
1849}
1850
1851unsigned fastEmit_AArch64ISD_UADDLP_r(MVT VT, MVT RetVT, unsigned Op0) {
1852 switch (VT.SimpleTy) {
1853 case MVT::v8i8: return fastEmit_AArch64ISD_UADDLP_MVT_v8i8_r(RetVT, Op0);
1854 case MVT::v16i8: return fastEmit_AArch64ISD_UADDLP_MVT_v16i8_r(RetVT, Op0);
1855 case MVT::v4i16: return fastEmit_AArch64ISD_UADDLP_MVT_v4i16_r(RetVT, Op0);
1856 case MVT::v8i16: return fastEmit_AArch64ISD_UADDLP_MVT_v8i16_r(RetVT, Op0);
1857 case MVT::v2i32: return fastEmit_AArch64ISD_UADDLP_MVT_v2i32_r(RetVT, Op0);
1858 case MVT::v4i32: return fastEmit_AArch64ISD_UADDLP_MVT_v4i32_r(RetVT, Op0);
1859 default: return 0;
1860 }
1861}
1862
1863// FastEmit functions for AArch64ISD::UITOF.
1864
1865unsigned fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, unsigned Op0) {
1866 if (RetVT.SimpleTy != MVT::f16)
1867 return 0;
1868 if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
1869 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i16, RC: &AArch64::FPR16RegClass, Op0);
1870 }
1871 return 0;
1872}
1873
1874unsigned fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, unsigned Op0) {
1875 if (RetVT.SimpleTy != MVT::f32)
1876 return 0;
1877 if ((Subtarget->hasNEON())) {
1878 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i32, RC: &AArch64::FPR32RegClass, Op0);
1879 }
1880 return 0;
1881}
1882
1883unsigned fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, unsigned Op0) {
1884 if (RetVT.SimpleTy != MVT::f64)
1885 return 0;
1886 if ((Subtarget->hasNEON())) {
1887 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv1i64, RC: &AArch64::FPR64RegClass, Op0);
1888 }
1889 return 0;
1890}
1891
1892unsigned fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, unsigned Op0) {
1893 switch (VT.SimpleTy) {
1894 case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0);
1895 case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0);
1896 case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0);
1897 default: return 0;
1898 }
1899}
1900
1901// FastEmit functions for AArch64ISD::UUNPKHI.
1902
1903unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) {
1904 if (RetVT.SimpleTy != MVT::nxv8i16)
1905 return 0;
1906 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1907 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1908 }
1909 return 0;
1910}
1911
1912unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) {
1913 if (RetVT.SimpleTy != MVT::nxv4i32)
1914 return 0;
1915 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1916 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1917 }
1918 return 0;
1919}
1920
1921unsigned fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) {
1922 if (RetVT.SimpleTy != MVT::nxv2i64)
1923 return 0;
1924 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1925 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKHI_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1926 }
1927 return 0;
1928}
1929
1930unsigned fastEmit_AArch64ISD_UUNPKHI_r(MVT VT, MVT RetVT, unsigned Op0) {
1931 switch (VT.SimpleTy) {
1932 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv16i8_r(RetVT, Op0);
1933 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv8i16_r(RetVT, Op0);
1934 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKHI_MVT_nxv4i32_r(RetVT, Op0);
1935 default: return 0;
1936 }
1937}
1938
1939// FastEmit functions for AArch64ISD::UUNPKLO.
1940
1941unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) {
1942 if (RetVT.SimpleTy != MVT::nxv8i16)
1943 return 0;
1944 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1945 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
1946 }
1947 return 0;
1948}
1949
1950unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) {
1951 if (RetVT.SimpleTy != MVT::nxv4i32)
1952 return 0;
1953 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1954 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
1955 }
1956 return 0;
1957}
1958
1959unsigned fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) {
1960 if (RetVT.SimpleTy != MVT::nxv2i64)
1961 return 0;
1962 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
1963 return fastEmitInst_r(MachineInstOpcode: AArch64::UUNPKLO_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
1964 }
1965 return 0;
1966}
1967
1968unsigned fastEmit_AArch64ISD_UUNPKLO_r(MVT VT, MVT RetVT, unsigned Op0) {
1969 switch (VT.SimpleTy) {
1970 case MVT::nxv16i8: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv16i8_r(RetVT, Op0);
1971 case MVT::nxv8i16: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv8i16_r(RetVT, Op0);
1972 case MVT::nxv4i32: return fastEmit_AArch64ISD_UUNPKLO_MVT_nxv4i32_r(RetVT, Op0);
1973 default: return 0;
1974 }
1975}
1976
1977// FastEmit functions for ISD::ABS.
1978
1979unsigned fastEmit_ISD_ABS_MVT_i32_r(MVT RetVT, unsigned Op0) {
1980 if (RetVT.SimpleTy != MVT::i32)
1981 return 0;
1982 if ((Subtarget->hasCSSC())) {
1983 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSWr, RC: &AArch64::GPR32RegClass, Op0);
1984 }
1985 return 0;
1986}
1987
1988unsigned fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, unsigned Op0) {
1989 if (RetVT.SimpleTy != MVT::i64)
1990 return 0;
1991 if ((!Subtarget->hasCSSC())) {
1992 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
1993 }
1994 if ((Subtarget->hasCSSC())) {
1995 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSXr, RC: &AArch64::GPR64RegClass, Op0);
1996 }
1997 return 0;
1998}
1999
2000unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
2001 if (RetVT.SimpleTy != MVT::v8i8)
2002 return 0;
2003 if ((Subtarget->isNeonAvailable())) {
2004 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i8, RC: &AArch64::FPR64RegClass, Op0);
2005 }
2006 return 0;
2007}
2008
2009unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
2010 if (RetVT.SimpleTy != MVT::v16i8)
2011 return 0;
2012 if ((Subtarget->isNeonAvailable())) {
2013 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv16i8, RC: &AArch64::FPR128RegClass, Op0);
2014 }
2015 return 0;
2016}
2017
2018unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
2019 if (RetVT.SimpleTy != MVT::v4i16)
2020 return 0;
2021 if ((Subtarget->isNeonAvailable())) {
2022 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i16, RC: &AArch64::FPR64RegClass, Op0);
2023 }
2024 return 0;
2025}
2026
2027unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2028 if (RetVT.SimpleTy != MVT::v8i16)
2029 return 0;
2030 if ((Subtarget->isNeonAvailable())) {
2031 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv8i16, RC: &AArch64::FPR128RegClass, Op0);
2032 }
2033 return 0;
2034}
2035
2036unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
2037 if (RetVT.SimpleTy != MVT::v2i32)
2038 return 0;
2039 if ((Subtarget->isNeonAvailable())) {
2040 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i32, RC: &AArch64::FPR64RegClass, Op0);
2041 }
2042 return 0;
2043}
2044
2045unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2046 if (RetVT.SimpleTy != MVT::v4i32)
2047 return 0;
2048 if ((Subtarget->isNeonAvailable())) {
2049 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv4i32, RC: &AArch64::FPR128RegClass, Op0);
2050 }
2051 return 0;
2052}
2053
2054unsigned fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
2055 if (RetVT.SimpleTy != MVT::v1i64)
2056 return 0;
2057 if ((Subtarget->isNeonAvailable())) {
2058 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv1i64, RC: &AArch64::FPR64RegClass, Op0);
2059 }
2060 return 0;
2061}
2062
2063unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2064 if (RetVT.SimpleTy != MVT::v2i64)
2065 return 0;
2066 if ((Subtarget->isNeonAvailable())) {
2067 return fastEmitInst_r(MachineInstOpcode: AArch64::ABSv2i64, RC: &AArch64::FPR128RegClass, Op0);
2068 }
2069 return 0;
2070}
2071
2072unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) {
2073 switch (VT.SimpleTy) {
2074 case MVT::i32: return fastEmit_ISD_ABS_MVT_i32_r(RetVT, Op0);
2075 case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0);
2076 case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
2077 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
2078 case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
2079 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
2080 case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
2081 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
2082 case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0);
2083 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0);
2084 default: return 0;
2085 }
2086}
2087
2088// FastEmit functions for ISD::BITCAST.
2089
2090unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0) {
2091 if ((!Subtarget->isLittleEndian())) {
2092 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2093 }
2094 return 0;
2095}
2096
2097unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0) {
2098 if ((!Subtarget->isLittleEndian())) {
2099 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2100 }
2101 return 0;
2102}
2103
2104unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0) {
2105 if ((!Subtarget->isLittleEndian())) {
2106 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2107 }
2108 return 0;
2109}
2110
2111unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0) {
2112 if ((!Subtarget->isLittleEndian())) {
2113 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2114 }
2115 return 0;
2116}
2117
2118unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(unsigned Op0) {
2119 if ((!Subtarget->isLittleEndian())) {
2120 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2121 }
2122 return 0;
2123}
2124
2125unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0) {
2126 if ((!Subtarget->isLittleEndian())) {
2127 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2128 }
2129 return 0;
2130}
2131
2132unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) {
2133switch (RetVT.SimpleTy) {
2134 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
2135 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
2136 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
2137 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
2138 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
2139 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
2140 default: return 0;
2141}
2142}
2143
2144unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0) {
2145 if ((!Subtarget->isLittleEndian())) {
2146 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2147 }
2148 return 0;
2149}
2150
2151unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0) {
2152 if ((!Subtarget->isLittleEndian())) {
2153 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2154 }
2155 return 0;
2156}
2157
2158unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0) {
2159 if ((!Subtarget->isLittleEndian())) {
2160 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2161 }
2162 return 0;
2163}
2164
2165unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0) {
2166 if ((!Subtarget->isLittleEndian())) {
2167 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2168 }
2169 return 0;
2170}
2171
2172unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0) {
2173 if ((!Subtarget->isLittleEndian())) {
2174 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2175 }
2176 return 0;
2177}
2178
2179unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(unsigned Op0) {
2180 if ((!Subtarget->isLittleEndian())) {
2181 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2182 }
2183 return 0;
2184}
2185
2186unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0) {
2187 if ((!Subtarget->isLittleEndian())) {
2188 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2189 }
2190 return 0;
2191}
2192
2193unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(unsigned Op0) {
2194 if ((!Subtarget->isLittleEndian())) {
2195 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2196 }
2197 return 0;
2198}
2199
2200unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
2201switch (RetVT.SimpleTy) {
2202 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
2203 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
2204 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
2205 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
2206 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
2207 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
2208 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
2209 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0);
2210 default: return 0;
2211}
2212}
2213
2214unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
2215 if ((!Subtarget->isLittleEndian())) {
2216 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2217 }
2218 return 0;
2219}
2220
2221unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
2222 if ((!Subtarget->isLittleEndian())) {
2223 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2224 }
2225 return 0;
2226}
2227
2228unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
2229 if ((!Subtarget->isLittleEndian())) {
2230 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2231 }
2232 return 0;
2233}
2234
2235unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0) {
2236 if ((!Subtarget->isLittleEndian())) {
2237 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2238 }
2239 return 0;
2240}
2241
2242unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(unsigned Op0) {
2243 if ((!Subtarget->isLittleEndian())) {
2244 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2245 }
2246 return 0;
2247}
2248
2249unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0) {
2250 if ((!Subtarget->isLittleEndian())) {
2251 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2252 }
2253 return 0;
2254}
2255
2256unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0) {
2257 if ((!Subtarget->isLittleEndian())) {
2258 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2259 }
2260 return 0;
2261}
2262
2263unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
2264switch (RetVT.SimpleTy) {
2265 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
2266 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
2267 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
2268 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
2269 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
2270 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
2271 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
2272 default: return 0;
2273}
2274}
2275
2276unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0) {
2277 if ((!Subtarget->isLittleEndian())) {
2278 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2279 }
2280 return 0;
2281}
2282
2283unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0) {
2284 if ((!Subtarget->isLittleEndian())) {
2285 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2286 }
2287 return 0;
2288}
2289
2290unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0) {
2291 if ((!Subtarget->isLittleEndian())) {
2292 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2293 }
2294 return 0;
2295}
2296
2297unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0) {
2298 if ((!Subtarget->isLittleEndian())) {
2299 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2300 }
2301 return 0;
2302}
2303
2304unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0) {
2305 if ((!Subtarget->isLittleEndian())) {
2306 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2307 }
2308 return 0;
2309}
2310
2311unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(unsigned Op0) {
2312 if ((!Subtarget->isLittleEndian())) {
2313 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2314 }
2315 return 0;
2316}
2317
2318unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
2319switch (RetVT.SimpleTy) {
2320 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
2321 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
2322 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
2323 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
2324 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
2325 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0);
2326 default: return 0;
2327}
2328}
2329
2330unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0) {
2331 if ((!Subtarget->isLittleEndian())) {
2332 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2333 }
2334 return 0;
2335}
2336
2337unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
2338 if ((!Subtarget->isLittleEndian())) {
2339 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2340 }
2341 return 0;
2342}
2343
2344unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
2345 if ((!Subtarget->isLittleEndian())) {
2346 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2347 }
2348 return 0;
2349}
2350
2351unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
2352 if ((!Subtarget->isLittleEndian())) {
2353 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2354 }
2355 return 0;
2356}
2357
2358unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0) {
2359 if ((!Subtarget->isLittleEndian())) {
2360 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2361 }
2362 return 0;
2363}
2364
2365unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
2366switch (RetVT.SimpleTy) {
2367 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
2368 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
2369 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
2370 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
2371 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
2372 default: return 0;
2373}
2374}
2375
2376unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0) {
2377 if ((!Subtarget->isLittleEndian())) {
2378 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2379 }
2380 return 0;
2381}
2382
2383unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0) {
2384 if ((!Subtarget->isLittleEndian())) {
2385 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2386 }
2387 return 0;
2388}
2389
2390unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0) {
2391 if ((!Subtarget->isLittleEndian())) {
2392 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2393 }
2394 return 0;
2395}
2396
2397unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0) {
2398 if ((!Subtarget->isLittleEndian())) {
2399 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2400 }
2401 return 0;
2402}
2403
2404unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0) {
2405 if ((!Subtarget->isLittleEndian())) {
2406 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2407 }
2408 return 0;
2409}
2410
2411unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(unsigned Op0) {
2412 if ((!Subtarget->isLittleEndian())) {
2413 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2414 }
2415 return 0;
2416}
2417
2418unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(unsigned Op0) {
2419 if ((!Subtarget->isLittleEndian())) {
2420 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2421 }
2422 return 0;
2423}
2424
2425unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
2426switch (RetVT.SimpleTy) {
2427 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
2428 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
2429 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
2430 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
2431 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
2432 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
2433 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0);
2434 default: return 0;
2435}
2436}
2437
2438unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
2439 if ((!Subtarget->isLittleEndian())) {
2440 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2441 }
2442 return 0;
2443}
2444
2445unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
2446 if ((!Subtarget->isLittleEndian())) {
2447 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2448 }
2449 return 0;
2450}
2451
2452unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0) {
2453 if ((!Subtarget->isLittleEndian())) {
2454 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2455 }
2456 return 0;
2457}
2458
2459unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
2460 if ((!Subtarget->isLittleEndian())) {
2461 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2462 }
2463 return 0;
2464}
2465
2466unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(unsigned Op0) {
2467 if ((!Subtarget->isLittleEndian())) {
2468 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2469 }
2470 return 0;
2471}
2472
2473unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
2474 if ((!Subtarget->isLittleEndian())) {
2475 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2476 }
2477 return 0;
2478}
2479
2480unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
2481switch (RetVT.SimpleTy) {
2482 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
2483 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
2484 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
2485 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
2486 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
2487 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
2488 default: return 0;
2489}
2490}
2491
2492unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0) {
2493 if ((!Subtarget->isLittleEndian())) {
2494 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2495 }
2496 return 0;
2497}
2498
2499unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0) {
2500 if ((!Subtarget->isLittleEndian())) {
2501 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2502 }
2503 return 0;
2504}
2505
2506unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0) {
2507 if ((!Subtarget->isLittleEndian())) {
2508 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2509 }
2510 return 0;
2511}
2512
2513unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0) {
2514 if ((!Subtarget->isLittleEndian())) {
2515 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2516 }
2517 return 0;
2518}
2519
2520unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(unsigned Op0) {
2521 if ((!Subtarget->isLittleEndian())) {
2522 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2523 }
2524 return 0;
2525}
2526
2527unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0) {
2528 if ((!Subtarget->isLittleEndian())) {
2529 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2530 }
2531 return 0;
2532}
2533
2534unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
2535switch (RetVT.SimpleTy) {
2536 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
2537 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
2538 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
2539 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
2540 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
2541 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
2542 default: return 0;
2543}
2544}
2545
2546unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
2547 if ((!Subtarget->isLittleEndian())) {
2548 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2549 }
2550 return 0;
2551}
2552
2553unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
2554 if ((!Subtarget->isLittleEndian())) {
2555 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2556 }
2557 return 0;
2558}
2559
2560unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
2561 if ((!Subtarget->isLittleEndian())) {
2562 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2563 }
2564 return 0;
2565}
2566
2567unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
2568 if ((!Subtarget->isLittleEndian())) {
2569 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2570 }
2571 return 0;
2572}
2573
2574unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(unsigned Op0) {
2575 if ((!Subtarget->isLittleEndian())) {
2576 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2577 }
2578 return 0;
2579}
2580
2581unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
2582 if ((!Subtarget->isLittleEndian())) {
2583 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2584 }
2585 return 0;
2586}
2587
2588unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
2589switch (RetVT.SimpleTy) {
2590 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
2591 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
2592 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
2593 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
2594 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
2595 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
2596 default: return 0;
2597}
2598}
2599
2600unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0) {
2601 if ((!Subtarget->isLittleEndian())) {
2602 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2603 }
2604 return 0;
2605}
2606
2607unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0) {
2608 if ((!Subtarget->isLittleEndian())) {
2609 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2610 }
2611 return 0;
2612}
2613
2614unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0) {
2615 if ((!Subtarget->isLittleEndian())) {
2616 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2617 }
2618 return 0;
2619}
2620
2621unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0) {
2622 if ((!Subtarget->isLittleEndian())) {
2623 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2624 }
2625 return 0;
2626}
2627
2628unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0) {
2629 if ((!Subtarget->isLittleEndian())) {
2630 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2631 }
2632 return 0;
2633}
2634
2635unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(unsigned Op0) {
2636 if ((!Subtarget->isLittleEndian())) {
2637 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2638 }
2639 return 0;
2640}
2641
2642unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
2643switch (RetVT.SimpleTy) {
2644 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
2645 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
2646 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
2647 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
2648 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
2649 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0);
2650 default: return 0;
2651}
2652}
2653
2654unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0) {
2655 if ((!Subtarget->isLittleEndian())) {
2656 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2657 }
2658 return 0;
2659}
2660
2661unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
2662 if ((!Subtarget->isLittleEndian())) {
2663 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2664 }
2665 return 0;
2666}
2667
2668unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
2669 if ((!Subtarget->isLittleEndian())) {
2670 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2671 }
2672 return 0;
2673}
2674
2675unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
2676 if ((!Subtarget->isLittleEndian())) {
2677 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2678 }
2679 return 0;
2680}
2681
2682unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
2683 if ((!Subtarget->isLittleEndian())) {
2684 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2685 }
2686 return 0;
2687}
2688
2689unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
2690switch (RetVT.SimpleTy) {
2691 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
2692 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
2693 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
2694 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
2695 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
2696 default: return 0;
2697}
2698}
2699
2700unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(unsigned Op0) {
2701 if ((!Subtarget->isLittleEndian())) {
2702 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2703 }
2704 return 0;
2705}
2706
2707unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(unsigned Op0) {
2708 if ((!Subtarget->isLittleEndian())) {
2709 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
2710 }
2711 return 0;
2712}
2713
2714unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(unsigned Op0) {
2715 if ((!Subtarget->isLittleEndian())) {
2716 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2717 }
2718 return 0;
2719}
2720
2721unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(unsigned Op0) {
2722 if ((!Subtarget->isLittleEndian())) {
2723 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2724 }
2725 return 0;
2726}
2727
2728unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(unsigned Op0) {
2729 if ((!Subtarget->isLittleEndian())) {
2730 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2731 }
2732 return 0;
2733}
2734
2735unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(unsigned Op0) {
2736 if ((!Subtarget->isLittleEndian())) {
2737 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2738 }
2739 return 0;
2740}
2741
2742unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
2743switch (RetVT.SimpleTy) {
2744 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
2745 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
2746 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
2747 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
2748 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
2749 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1f64_r(Op0);
2750 default: return 0;
2751}
2752}
2753
2754unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(unsigned Op0) {
2755 if ((!Subtarget->isLittleEndian())) {
2756 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
2757 }
2758 return 0;
2759}
2760
2761unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(unsigned Op0) {
2762 if ((!Subtarget->isLittleEndian())) {
2763 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2764 }
2765 return 0;
2766}
2767
2768unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(unsigned Op0) {
2769 if ((!Subtarget->isLittleEndian())) {
2770 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2771 }
2772 return 0;
2773}
2774
2775unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(unsigned Op0) {
2776 if ((!Subtarget->isLittleEndian())) {
2777 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2778 }
2779 return 0;
2780}
2781
2782unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(unsigned Op0) {
2783 if ((!Subtarget->isLittleEndian())) {
2784 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2785 }
2786 return 0;
2787}
2788
2789unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
2790switch (RetVT.SimpleTy) {
2791 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
2792 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
2793 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
2794 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
2795 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
2796 default: return 0;
2797}
2798}
2799
2800unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0) {
2801 if ((!Subtarget->isLittleEndian())) {
2802 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2803 }
2804 return 0;
2805}
2806
2807unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0) {
2808 if ((!Subtarget->isLittleEndian())) {
2809 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
2810 }
2811 return 0;
2812}
2813
2814unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0) {
2815 if ((!Subtarget->isLittleEndian())) {
2816 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2817 }
2818 return 0;
2819}
2820
2821unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0) {
2822 if ((!Subtarget->isLittleEndian())) {
2823 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2824 }
2825 return 0;
2826}
2827
2828unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0) {
2829 if ((!Subtarget->isLittleEndian())) {
2830 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2831 }
2832 return 0;
2833}
2834
2835unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(unsigned Op0) {
2836 if ((!Subtarget->isLittleEndian())) {
2837 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v4i16, RC: &AArch64::FPR64RegClass, Op0);
2838 }
2839 return 0;
2840}
2841
2842unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(unsigned Op0) {
2843 if ((!Subtarget->isLittleEndian())) {
2844 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2845 }
2846 return 0;
2847}
2848
2849unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
2850switch (RetVT.SimpleTy) {
2851 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
2852 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
2853 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
2854 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
2855 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
2856 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
2857 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0);
2858 default: return 0;
2859}
2860}
2861
2862unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0) {
2863 if ((!Subtarget->isLittleEndian())) {
2864 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
2865 }
2866 return 0;
2867}
2868
2869unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0) {
2870 if ((!Subtarget->isLittleEndian())) {
2871 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2872 }
2873 return 0;
2874}
2875
2876unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
2877 if ((!Subtarget->isLittleEndian())) {
2878 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2879 }
2880 return 0;
2881}
2882
2883unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0) {
2884 if ((!Subtarget->isLittleEndian())) {
2885 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2886 }
2887 return 0;
2888}
2889
2890unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(unsigned Op0) {
2891 if ((!Subtarget->isLittleEndian())) {
2892 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i16, RC: &AArch64::FPR128RegClass, Op0);
2893 }
2894 return 0;
2895}
2896
2897unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0) {
2898 if ((!Subtarget->isLittleEndian())) {
2899 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2900 }
2901 return 0;
2902}
2903
2904unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
2905switch (RetVT.SimpleTy) {
2906 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
2907 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
2908 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
2909 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
2910 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
2911 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
2912 default: return 0;
2913}
2914}
2915
2916unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(unsigned Op0) {
2917 if ((!Subtarget->isLittleEndian())) {
2918 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i8, RC: &AArch64::FPR64RegClass, Op0);
2919 }
2920 return 0;
2921}
2922
2923unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(unsigned Op0) {
2924 if ((!Subtarget->isLittleEndian())) {
2925 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2926 }
2927 return 0;
2928}
2929
2930unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(unsigned Op0) {
2931 if ((!Subtarget->isLittleEndian())) {
2932 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2933 }
2934 return 0;
2935}
2936
2937unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(unsigned Op0) {
2938 if ((!Subtarget->isLittleEndian())) {
2939 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2940 }
2941 return 0;
2942}
2943
2944unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(unsigned Op0) {
2945 if ((!Subtarget->isLittleEndian())) {
2946 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i16, RC: &AArch64::FPR64RegClass, Op0);
2947 }
2948 return 0;
2949}
2950
2951unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(unsigned Op0) {
2952 if ((!Subtarget->isLittleEndian())) {
2953 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v2i32, RC: &AArch64::FPR64RegClass, Op0);
2954 }
2955 return 0;
2956}
2957
2958unsigned fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, unsigned Op0) {
2959switch (RetVT.SimpleTy) {
2960 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0);
2961 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0);
2962 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0);
2963 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0);
2964 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4bf16_r(Op0);
2965 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0);
2966 default: return 0;
2967}
2968}
2969
2970unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0) {
2971 if ((!Subtarget->isLittleEndian())) {
2972 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
2973 }
2974 return 0;
2975}
2976
2977unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0) {
2978 if ((!Subtarget->isLittleEndian())) {
2979 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2980 }
2981 return 0;
2982}
2983
2984unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
2985 if ((!Subtarget->isLittleEndian())) {
2986 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
2987 }
2988 return 0;
2989}
2990
2991unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
2992 if ((!Subtarget->isLittleEndian())) {
2993 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
2994 }
2995 return 0;
2996}
2997
2998unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(unsigned Op0) {
2999 if ((!Subtarget->isLittleEndian())) {
3000 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v8i16, RC: &AArch64::FPR128RegClass, Op0);
3001 }
3002 return 0;
3003}
3004
3005unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
3006 if ((!Subtarget->isLittleEndian())) {
3007 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v4i32, RC: &AArch64::FPR128RegClass, Op0);
3008 }
3009 return 0;
3010}
3011
3012unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3013switch (RetVT.SimpleTy) {
3014 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
3015 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
3016 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
3017 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
3018 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
3019 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
3020 default: return 0;
3021}
3022}
3023
3024unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
3025 switch (VT.SimpleTy) {
3026 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
3027 case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
3028 case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
3029 case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
3030 case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
3031 case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
3032 case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
3033 case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
3034 case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
3035 case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
3036 case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
3037 case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
3038 case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
3039 case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
3040 case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
3041 case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0);
3042 case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
3043 default: return 0;
3044 }
3045}
3046
3047// FastEmit functions for ISD::BITREVERSE.
3048
3049unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0) {
3050 if (RetVT.SimpleTy != MVT::i32)
3051 return 0;
3052 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITWr, RC: &AArch64::GPR32RegClass, Op0);
3053}
3054
3055unsigned fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, unsigned Op0) {
3056 if (RetVT.SimpleTy != MVT::i64)
3057 return 0;
3058 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITXr, RC: &AArch64::GPR64RegClass, Op0);
3059}
3060
3061unsigned fastEmit_ISD_BITREVERSE_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
3062 if (RetVT.SimpleTy != MVT::v8i8)
3063 return 0;
3064 if ((Subtarget->isNeonAvailable())) {
3065 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv8i8, RC: &AArch64::FPR64RegClass, Op0);
3066 }
3067 return 0;
3068}
3069
3070unsigned fastEmit_ISD_BITREVERSE_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3071 if (RetVT.SimpleTy != MVT::v16i8)
3072 return 0;
3073 if ((Subtarget->isNeonAvailable())) {
3074 return fastEmitInst_r(MachineInstOpcode: AArch64::RBITv16i8, RC: &AArch64::FPR128RegClass, Op0);
3075 }
3076 return 0;
3077}
3078
3079unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0) {
3080 switch (VT.SimpleTy) {
3081 case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
3082 case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0);
3083 case MVT::v8i8: return fastEmit_ISD_BITREVERSE_MVT_v8i8_r(RetVT, Op0);
3084 case MVT::v16i8: return fastEmit_ISD_BITREVERSE_MVT_v16i8_r(RetVT, Op0);
3085 default: return 0;
3086 }
3087}
3088
3089// FastEmit functions for ISD::BRIND.
3090
3091unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0) {
3092 if (RetVT.SimpleTy != MVT::isVoid)
3093 return 0;
3094 return fastEmitInst_r(MachineInstOpcode: AArch64::BR, RC: &AArch64::GPR64RegClass, Op0);
3095}
3096
3097unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
3098 switch (VT.SimpleTy) {
3099 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0);
3100 default: return 0;
3101 }
3102}
3103
3104// FastEmit functions for ISD::BSWAP.
3105
3106unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) {
3107 if (RetVT.SimpleTy != MVT::i32)
3108 return 0;
3109 return fastEmitInst_r(MachineInstOpcode: AArch64::REVWr, RC: &AArch64::GPR32RegClass, Op0);
3110}
3111
3112unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0) {
3113 if (RetVT.SimpleTy != MVT::i64)
3114 return 0;
3115 return fastEmitInst_r(MachineInstOpcode: AArch64::REVXr, RC: &AArch64::GPR64RegClass, Op0);
3116}
3117
3118unsigned fastEmit_ISD_BSWAP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
3119 if (RetVT.SimpleTy != MVT::v4i16)
3120 return 0;
3121 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v8i8, RC: &AArch64::FPR64RegClass, Op0);
3122}
3123
3124unsigned fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3125 if (RetVT.SimpleTy != MVT::v8i16)
3126 return 0;
3127 return fastEmitInst_r(MachineInstOpcode: AArch64::REV16v16i8, RC: &AArch64::FPR128RegClass, Op0);
3128}
3129
3130unsigned fastEmit_ISD_BSWAP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
3131 if (RetVT.SimpleTy != MVT::v2i32)
3132 return 0;
3133 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v8i8, RC: &AArch64::FPR64RegClass, Op0);
3134}
3135
3136unsigned fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3137 if (RetVT.SimpleTy != MVT::v4i32)
3138 return 0;
3139 return fastEmitInst_r(MachineInstOpcode: AArch64::REV32v16i8, RC: &AArch64::FPR128RegClass, Op0);
3140}
3141
3142unsigned fastEmit_ISD_BSWAP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
3143 if (RetVT.SimpleTy != MVT::v2i64)
3144 return 0;
3145 return fastEmitInst_r(MachineInstOpcode: AArch64::REV64v16i8, RC: &AArch64::FPR128RegClass, Op0);
3146}
3147
3148unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) {
3149 switch (VT.SimpleTy) {
3150 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
3151 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0);
3152 case MVT::v4i16: return fastEmit_ISD_BSWAP_MVT_v4i16_r(RetVT, Op0);
3153 case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
3154 case MVT::v2i32: return fastEmit_ISD_BSWAP_MVT_v2i32_r(RetVT, Op0);
3155 case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
3156 case MVT::v2i64: return fastEmit_ISD_BSWAP_MVT_v2i64_r(RetVT, Op0);
3157 default: return 0;
3158 }
3159}
3160
3161// FastEmit functions for ISD::CTLZ.
3162
3163unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
3164 if (RetVT.SimpleTy != MVT::i32)
3165 return 0;
3166 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZWr, RC: &AArch64::GPR32RegClass, Op0);
3167}
3168
3169unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0) {
3170 if (RetVT.SimpleTy != MVT::i64)
3171 return 0;
3172 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZXr, RC: &AArch64::GPR64RegClass, Op0);
3173}
3174
3175unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
3176 if (RetVT.SimpleTy != MVT::v8i8)
3177 return 0;
3178 if ((Subtarget->isNeonAvailable())) {
3179 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i8, RC: &AArch64::FPR64RegClass, Op0);
3180 }
3181 return 0;
3182}
3183
3184unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3185 if (RetVT.SimpleTy != MVT::v16i8)
3186 return 0;
3187 if ((Subtarget->isNeonAvailable())) {
3188 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv16i8, RC: &AArch64::FPR128RegClass, Op0);
3189 }
3190 return 0;
3191}
3192
3193unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
3194 if (RetVT.SimpleTy != MVT::v4i16)
3195 return 0;
3196 if ((Subtarget->isNeonAvailable())) {
3197 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i16, RC: &AArch64::FPR64RegClass, Op0);
3198 }
3199 return 0;
3200}
3201
3202unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
3203 if (RetVT.SimpleTy != MVT::v8i16)
3204 return 0;
3205 if ((Subtarget->isNeonAvailable())) {
3206 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv8i16, RC: &AArch64::FPR128RegClass, Op0);
3207 }
3208 return 0;
3209}
3210
3211unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
3212 if (RetVT.SimpleTy != MVT::v2i32)
3213 return 0;
3214 if ((Subtarget->isNeonAvailable())) {
3215 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv2i32, RC: &AArch64::FPR64RegClass, Op0);
3216 }
3217 return 0;
3218}
3219
3220unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
3221 if (RetVT.SimpleTy != MVT::v4i32)
3222 return 0;
3223 if ((Subtarget->isNeonAvailable())) {
3224 return fastEmitInst_r(MachineInstOpcode: AArch64::CLZv4i32, RC: &AArch64::FPR128RegClass, Op0);
3225 }
3226 return 0;
3227}
3228
3229unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) {
3230 switch (VT.SimpleTy) {
3231 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
3232 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0);
3233 case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
3234 case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
3235 case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
3236 case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
3237 case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
3238 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
3239 default: return 0;
3240 }
3241}
3242
3243// FastEmit functions for ISD::CTPOP.
3244
3245unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0) {
3246 if (RetVT.SimpleTy != MVT::i32)
3247 return 0;
3248 if ((Subtarget->hasCSSC())) {
3249 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTWr, RC: &AArch64::GPR32RegClass, Op0);
3250 }
3251 return 0;
3252}
3253
3254unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0) {
3255 if (RetVT.SimpleTy != MVT::i64)
3256 return 0;
3257 if ((Subtarget->hasCSSC())) {
3258 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTXr, RC: &AArch64::GPR64RegClass, Op0);
3259 }
3260 return 0;
3261}
3262
3263unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
3264 if (RetVT.SimpleTy != MVT::v8i8)
3265 return 0;
3266 if ((Subtarget->isNeonAvailable())) {
3267 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv8i8, RC: &AArch64::FPR64RegClass, Op0);
3268 }
3269 return 0;
3270}
3271
3272unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
3273 if (RetVT.SimpleTy != MVT::v16i8)
3274 return 0;
3275 if ((Subtarget->isNeonAvailable())) {
3276 return fastEmitInst_r(MachineInstOpcode: AArch64::CNTv16i8, RC: &AArch64::FPR128RegClass, Op0);
3277 }
3278 return 0;
3279}
3280
3281unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) {
3282 switch (VT.SimpleTy) {
3283 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0);
3284 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0);
3285 case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
3286 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
3287 default: return 0;
3288 }
3289}
3290
3291// FastEmit functions for ISD::CTTZ.
3292
3293unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
3294 if (RetVT.SimpleTy != MVT::i32)
3295 return 0;
3296 if ((Subtarget->hasCSSC())) {
3297 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZWr, RC: &AArch64::GPR32RegClass, Op0);
3298 }
3299 return 0;
3300}
3301
3302unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0) {
3303 if (RetVT.SimpleTy != MVT::i64)
3304 return 0;
3305 if ((Subtarget->hasCSSC())) {
3306 return fastEmitInst_r(MachineInstOpcode: AArch64::CTZXr, RC: &AArch64::GPR64RegClass, Op0);
3307 }
3308 return 0;
3309}
3310
3311unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0) {
3312 switch (VT.SimpleTy) {
3313 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0);
3314 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0);
3315 default: return 0;
3316 }
3317}
3318
3319// FastEmit functions for ISD::FABS.
3320
3321unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0) {
3322 if (RetVT.SimpleTy != MVT::f16)
3323 return 0;
3324 if ((Subtarget->hasFullFP16())) {
3325 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSHr, RC: &AArch64::FPR16RegClass, Op0);
3326 }
3327 return 0;
3328}
3329
3330unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) {
3331 if (RetVT.SimpleTy != MVT::f32)
3332 return 0;
3333 if ((Subtarget->hasFPARMv8())) {
3334 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSSr, RC: &AArch64::FPR32RegClass, Op0);
3335 }
3336 return 0;
3337}
3338
3339unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) {
3340 if (RetVT.SimpleTy != MVT::f64)
3341 return 0;
3342 if ((Subtarget->hasFPARMv8())) {
3343 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSDr, RC: &AArch64::FPR64RegClass, Op0);
3344 }
3345 return 0;
3346}
3347
3348unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3349 if (RetVT.SimpleTy != MVT::v4f16)
3350 return 0;
3351 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3352 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3353 }
3354 return 0;
3355}
3356
3357unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3358 if (RetVT.SimpleTy != MVT::v8f16)
3359 return 0;
3360 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3361 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3362 }
3363 return 0;
3364}
3365
3366unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3367 if (RetVT.SimpleTy != MVT::v2f32)
3368 return 0;
3369 if ((Subtarget->isNeonAvailable())) {
3370 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3371 }
3372 return 0;
3373}
3374
3375unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3376 if (RetVT.SimpleTy != MVT::v4f32)
3377 return 0;
3378 if ((Subtarget->isNeonAvailable())) {
3379 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3380 }
3381 return 0;
3382}
3383
3384unsigned fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3385 if (RetVT.SimpleTy != MVT::v2f64)
3386 return 0;
3387 if ((Subtarget->isNeonAvailable())) {
3388 return fastEmitInst_r(MachineInstOpcode: AArch64::FABSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3389 }
3390 return 0;
3391}
3392
3393unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) {
3394 switch (VT.SimpleTy) {
3395 case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
3396 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
3397 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
3398 case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
3399 case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
3400 case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
3401 case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
3402 case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0);
3403 default: return 0;
3404 }
3405}
3406
3407// FastEmit functions for ISD::FCEIL.
3408
3409unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0) {
3410 if (RetVT.SimpleTy != MVT::f16)
3411 return 0;
3412 if ((Subtarget->hasFullFP16())) {
3413 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
3414 }
3415 return 0;
3416}
3417
3418unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0) {
3419 if (RetVT.SimpleTy != MVT::f32)
3420 return 0;
3421 if ((Subtarget->hasFPARMv8())) {
3422 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
3423 }
3424 return 0;
3425}
3426
3427unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0) {
3428 if (RetVT.SimpleTy != MVT::f64)
3429 return 0;
3430 if ((Subtarget->hasFPARMv8())) {
3431 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
3432 }
3433 return 0;
3434}
3435
3436unsigned fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3437 if (RetVT.SimpleTy != MVT::v4f16)
3438 return 0;
3439 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3440 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
3441 }
3442 return 0;
3443}
3444
3445unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3446 if (RetVT.SimpleTy != MVT::v8f16)
3447 return 0;
3448 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3449 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
3450 }
3451 return 0;
3452}
3453
3454unsigned fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3455 if (RetVT.SimpleTy != MVT::v2f32)
3456 return 0;
3457 if ((Subtarget->isNeonAvailable())) {
3458 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
3459 }
3460 return 0;
3461}
3462
3463unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3464 if (RetVT.SimpleTy != MVT::v4f32)
3465 return 0;
3466 if ((Subtarget->isNeonAvailable())) {
3467 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
3468 }
3469 return 0;
3470}
3471
3472unsigned fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3473 if (RetVT.SimpleTy != MVT::v2f64)
3474 return 0;
3475 if ((Subtarget->isNeonAvailable())) {
3476 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
3477 }
3478 return 0;
3479}
3480
3481unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0) {
3482 switch (VT.SimpleTy) {
3483 case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
3484 case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
3485 case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
3486 case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0);
3487 case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
3488 case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0);
3489 case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
3490 case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0);
3491 default: return 0;
3492 }
3493}
3494
3495// FastEmit functions for ISD::FFLOOR.
3496
3497unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0) {
3498 if (RetVT.SimpleTy != MVT::f16)
3499 return 0;
3500 if ((Subtarget->hasFullFP16())) {
3501 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
3502 }
3503 return 0;
3504}
3505
3506unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0) {
3507 if (RetVT.SimpleTy != MVT::f32)
3508 return 0;
3509 if ((Subtarget->hasFPARMv8())) {
3510 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
3511 }
3512 return 0;
3513}
3514
3515unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0) {
3516 if (RetVT.SimpleTy != MVT::f64)
3517 return 0;
3518 if ((Subtarget->hasFPARMv8())) {
3519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
3520 }
3521 return 0;
3522}
3523
3524unsigned fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3525 if (RetVT.SimpleTy != MVT::v4f16)
3526 return 0;
3527 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
3529 }
3530 return 0;
3531}
3532
3533unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3534 if (RetVT.SimpleTy != MVT::v8f16)
3535 return 0;
3536 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
3538 }
3539 return 0;
3540}
3541
3542unsigned fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3543 if (RetVT.SimpleTy != MVT::v2f32)
3544 return 0;
3545 if ((Subtarget->isNeonAvailable())) {
3546 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
3547 }
3548 return 0;
3549}
3550
3551unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3552 if (RetVT.SimpleTy != MVT::v4f32)
3553 return 0;
3554 if ((Subtarget->isNeonAvailable())) {
3555 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
3556 }
3557 return 0;
3558}
3559
3560unsigned fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3561 if (RetVT.SimpleTy != MVT::v2f64)
3562 return 0;
3563 if ((Subtarget->isNeonAvailable())) {
3564 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
3565 }
3566 return 0;
3567}
3568
3569unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0) {
3570 switch (VT.SimpleTy) {
3571 case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
3572 case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
3573 case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
3574 case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0);
3575 case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
3576 case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0);
3577 case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
3578 case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0);
3579 default: return 0;
3580 }
3581}
3582
3583// FastEmit functions for ISD::FNEARBYINT.
3584
3585unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
3586 if (RetVT.SimpleTy != MVT::f16)
3587 return 0;
3588 if ((Subtarget->hasFullFP16())) {
3589 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
3590 }
3591 return 0;
3592}
3593
3594unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
3595 if (RetVT.SimpleTy != MVT::f32)
3596 return 0;
3597 if ((Subtarget->hasFPARMv8())) {
3598 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
3599 }
3600 return 0;
3601}
3602
3603unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
3604 if (RetVT.SimpleTy != MVT::f64)
3605 return 0;
3606 if ((Subtarget->hasFPARMv8())) {
3607 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
3608 }
3609 return 0;
3610}
3611
3612unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3613 if (RetVT.SimpleTy != MVT::v4f16)
3614 return 0;
3615 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3616 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
3617 }
3618 return 0;
3619}
3620
3621unsigned fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3622 if (RetVT.SimpleTy != MVT::v8f16)
3623 return 0;
3624 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3625 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
3626 }
3627 return 0;
3628}
3629
3630unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3631 if (RetVT.SimpleTy != MVT::v2f32)
3632 return 0;
3633 if ((Subtarget->isNeonAvailable())) {
3634 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
3635 }
3636 return 0;
3637}
3638
3639unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3640 if (RetVT.SimpleTy != MVT::v4f32)
3641 return 0;
3642 if ((Subtarget->isNeonAvailable())) {
3643 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
3644 }
3645 return 0;
3646}
3647
3648unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3649 if (RetVT.SimpleTy != MVT::v2f64)
3650 return 0;
3651 if ((Subtarget->isNeonAvailable())) {
3652 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
3653 }
3654 return 0;
3655}
3656
3657unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0) {
3658 switch (VT.SimpleTy) {
3659 case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
3660 case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
3661 case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
3662 case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
3663 case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
3664 case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
3665 case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
3666 case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
3667 default: return 0;
3668 }
3669}
3670
3671// FastEmit functions for ISD::FNEG.
3672
3673unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0) {
3674 if (RetVT.SimpleTy != MVT::f16)
3675 return 0;
3676 if ((Subtarget->hasFullFP16())) {
3677 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGHr, RC: &AArch64::FPR16RegClass, Op0);
3678 }
3679 return 0;
3680}
3681
3682unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) {
3683 if (RetVT.SimpleTy != MVT::f32)
3684 return 0;
3685 if ((Subtarget->hasFPARMv8())) {
3686 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGSr, RC: &AArch64::FPR32RegClass, Op0);
3687 }
3688 return 0;
3689}
3690
3691unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) {
3692 if (RetVT.SimpleTy != MVT::f64)
3693 return 0;
3694 if ((Subtarget->hasFPARMv8())) {
3695 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGDr, RC: &AArch64::FPR64RegClass, Op0);
3696 }
3697 return 0;
3698}
3699
3700unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3701 if (RetVT.SimpleTy != MVT::v4f16)
3702 return 0;
3703 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3704 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f16, RC: &AArch64::FPR64RegClass, Op0);
3705 }
3706 return 0;
3707}
3708
3709unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3710 if (RetVT.SimpleTy != MVT::v8f16)
3711 return 0;
3712 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3713 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv8f16, RC: &AArch64::FPR128RegClass, Op0);
3714 }
3715 return 0;
3716}
3717
3718unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3719 if (RetVT.SimpleTy != MVT::v2f32)
3720 return 0;
3721 if ((Subtarget->isNeonAvailable())) {
3722 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f32, RC: &AArch64::FPR64RegClass, Op0);
3723 }
3724 return 0;
3725}
3726
3727unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3728 if (RetVT.SimpleTy != MVT::v4f32)
3729 return 0;
3730 if ((Subtarget->isNeonAvailable())) {
3731 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv4f32, RC: &AArch64::FPR128RegClass, Op0);
3732 }
3733 return 0;
3734}
3735
3736unsigned fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3737 if (RetVT.SimpleTy != MVT::v2f64)
3738 return 0;
3739 if ((Subtarget->isNeonAvailable())) {
3740 return fastEmitInst_r(MachineInstOpcode: AArch64::FNEGv2f64, RC: &AArch64::FPR128RegClass, Op0);
3741 }
3742 return 0;
3743}
3744
3745unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) {
3746 switch (VT.SimpleTy) {
3747 case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
3748 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
3749 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
3750 case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
3751 case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
3752 case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
3753 case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
3754 case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0);
3755 default: return 0;
3756 }
3757}
3758
3759// FastEmit functions for ISD::FP_EXTEND.
3760
3761unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0) {
3762 if ((Subtarget->hasFPARMv8())) {
3763 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
3764 }
3765 return 0;
3766}
3767
3768unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0) {
3769 if ((Subtarget->hasFPARMv8())) {
3770 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
3771 }
3772 return 0;
3773}
3774
3775unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0) {
3776switch (RetVT.SimpleTy) {
3777 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
3778 case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
3779 default: return 0;
3780}
3781}
3782
3783unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
3784 if (RetVT.SimpleTy != MVT::f64)
3785 return 0;
3786 if ((Subtarget->hasFPARMv8())) {
3787 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
3788 }
3789 return 0;
3790}
3791
3792unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3793 if (RetVT.SimpleTy != MVT::v4f32)
3794 return 0;
3795 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3796}
3797
3798unsigned fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
3799 if (RetVT.SimpleTy != MVT::v4f32)
3800 return 0;
3801 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
3802}
3803
3804unsigned fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3805 if (RetVT.SimpleTy != MVT::v2f64)
3806 return 0;
3807 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
3808}
3809
3810unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
3811 switch (VT.SimpleTy) {
3812 case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0);
3813 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);
3814 case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
3815 case MVT::v4bf16: return fastEmit_ISD_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
3816 case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
3817 default: return 0;
3818 }
3819}
3820
3821// FastEmit functions for ISD::FP_ROUND.
3822
3823unsigned fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(unsigned Op0) {
3824 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
3825 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
3826 }
3827 return 0;
3828}
3829
3830unsigned fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(unsigned Op0) {
3831 if ((Subtarget->hasFPARMv8())) {
3832 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
3833 }
3834 return 0;
3835}
3836
3837unsigned fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
3838switch (RetVT.SimpleTy) {
3839 case MVT::bf16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
3840 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
3841 default: return 0;
3842}
3843}
3844
3845unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0) {
3846 if ((Subtarget->hasFPARMv8())) {
3847 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
3848 }
3849 return 0;
3850}
3851
3852unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0) {
3853 if ((Subtarget->hasFPARMv8())) {
3854 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
3855 }
3856 return 0;
3857}
3858
3859unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
3860switch (RetVT.SimpleTy) {
3861 case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
3862 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
3863 default: return 0;
3864}
3865}
3866
3867unsigned fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3868 if (RetVT.SimpleTy != MVT::v4f16)
3869 return 0;
3870 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
3871}
3872
3873unsigned fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3874 if (RetVT.SimpleTy != MVT::v2f32)
3875 return 0;
3876 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
3877}
3878
3879unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
3880 switch (VT.SimpleTy) {
3881 case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0);
3882 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0);
3883 case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
3884 case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
3885 default: return 0;
3886 }
3887}
3888
3889// FastEmit functions for ISD::FP_TO_SINT.
3890
3891unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) {
3892 if ((Subtarget->hasFullFP16())) {
3893 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
3894 }
3895 return 0;
3896}
3897
3898unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) {
3899 if ((Subtarget->hasFullFP16())) {
3900 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
3901 }
3902 return 0;
3903}
3904
3905unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
3906switch (RetVT.SimpleTy) {
3907 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
3908 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
3909 default: return 0;
3910}
3911}
3912
3913unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) {
3914 if ((Subtarget->hasFPARMv8())) {
3915 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
3916 }
3917 return 0;
3918}
3919
3920unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) {
3921 if ((Subtarget->hasFPARMv8())) {
3922 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
3923 }
3924 return 0;
3925}
3926
3927unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
3928switch (RetVT.SimpleTy) {
3929 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
3930 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
3931 default: return 0;
3932}
3933}
3934
3935unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) {
3936 if ((Subtarget->hasFPARMv8())) {
3937 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
3938 }
3939 return 0;
3940}
3941
3942unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) {
3943 if ((Subtarget->hasFPARMv8())) {
3944 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
3945 }
3946 return 0;
3947}
3948
3949unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
3950switch (RetVT.SimpleTy) {
3951 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
3952 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
3953 default: return 0;
3954}
3955}
3956
3957unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
3958 if (RetVT.SimpleTy != MVT::v4i16)
3959 return 0;
3960 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3961 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
3962 }
3963 return 0;
3964}
3965
3966unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
3967 if (RetVT.SimpleTy != MVT::v8i16)
3968 return 0;
3969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
3970 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
3971 }
3972 return 0;
3973}
3974
3975unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
3976 if (RetVT.SimpleTy != MVT::v2i32)
3977 return 0;
3978 if ((Subtarget->isNeonAvailable())) {
3979 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
3980 }
3981 return 0;
3982}
3983
3984unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
3985 if (RetVT.SimpleTy != MVT::v4i32)
3986 return 0;
3987 if ((Subtarget->isNeonAvailable())) {
3988 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
3989 }
3990 return 0;
3991}
3992
3993unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
3994 if (RetVT.SimpleTy != MVT::v2i64)
3995 return 0;
3996 if ((Subtarget->isNeonAvailable())) {
3997 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
3998 }
3999 return 0;
4000}
4001
4002unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
4003 switch (VT.SimpleTy) {
4004 case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
4005 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
4006 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
4007 case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
4008 case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
4009 case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
4010 case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
4011 case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
4012 default: return 0;
4013 }
4014}
4015
4016// FastEmit functions for ISD::FP_TO_UINT.
4017
4018unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) {
4019 if ((Subtarget->hasFullFP16())) {
4020 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
4021 }
4022 return 0;
4023}
4024
4025unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) {
4026 if ((Subtarget->hasFullFP16())) {
4027 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
4028 }
4029 return 0;
4030}
4031
4032unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
4033switch (RetVT.SimpleTy) {
4034 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
4035 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
4036 default: return 0;
4037}
4038}
4039
4040unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) {
4041 if ((Subtarget->hasFPARMv8())) {
4042 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
4043 }
4044 return 0;
4045}
4046
4047unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) {
4048 if ((Subtarget->hasFPARMv8())) {
4049 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
4050 }
4051 return 0;
4052}
4053
4054unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
4055switch (RetVT.SimpleTy) {
4056 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
4057 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
4058 default: return 0;
4059}
4060}
4061
4062unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) {
4063 if ((Subtarget->hasFPARMv8())) {
4064 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
4065 }
4066 return 0;
4067}
4068
4069unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) {
4070 if ((Subtarget->hasFPARMv8())) {
4071 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
4072 }
4073 return 0;
4074}
4075
4076unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
4077switch (RetVT.SimpleTy) {
4078 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
4079 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
4080 default: return 0;
4081}
4082}
4083
4084unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4085 if (RetVT.SimpleTy != MVT::v4i16)
4086 return 0;
4087 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4088 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
4089 }
4090 return 0;
4091}
4092
4093unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4094 if (RetVT.SimpleTy != MVT::v8i16)
4095 return 0;
4096 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4097 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
4098 }
4099 return 0;
4100}
4101
4102unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4103 if (RetVT.SimpleTy != MVT::v2i32)
4104 return 0;
4105 if ((Subtarget->isNeonAvailable())) {
4106 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
4107 }
4108 return 0;
4109}
4110
4111unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4112 if (RetVT.SimpleTy != MVT::v4i32)
4113 return 0;
4114 if ((Subtarget->isNeonAvailable())) {
4115 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
4116 }
4117 return 0;
4118}
4119
4120unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4121 if (RetVT.SimpleTy != MVT::v2i64)
4122 return 0;
4123 if ((Subtarget->isNeonAvailable())) {
4124 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
4125 }
4126 return 0;
4127}
4128
4129unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
4130 switch (VT.SimpleTy) {
4131 case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
4132 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
4133 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
4134 case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
4135 case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
4136 case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
4137 case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
4138 case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
4139 default: return 0;
4140 }
4141}
4142
4143// FastEmit functions for ISD::FRINT.
4144
4145unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
4146 if (RetVT.SimpleTy != MVT::f16)
4147 return 0;
4148 if ((Subtarget->hasFullFP16())) {
4149 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
4150 }
4151 return 0;
4152}
4153
4154unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
4155 if (RetVT.SimpleTy != MVT::f32)
4156 return 0;
4157 if ((Subtarget->hasFPARMv8())) {
4158 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
4159 }
4160 return 0;
4161}
4162
4163unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
4164 if (RetVT.SimpleTy != MVT::f64)
4165 return 0;
4166 if ((Subtarget->hasFPARMv8())) {
4167 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
4168 }
4169 return 0;
4170}
4171
4172unsigned fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4173 if (RetVT.SimpleTy != MVT::v4f16)
4174 return 0;
4175 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4176 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
4177 }
4178 return 0;
4179}
4180
4181unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4182 if (RetVT.SimpleTy != MVT::v8f16)
4183 return 0;
4184 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4185 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
4186 }
4187 return 0;
4188}
4189
4190unsigned fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4191 if (RetVT.SimpleTy != MVT::v2f32)
4192 return 0;
4193 if ((Subtarget->isNeonAvailable())) {
4194 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
4195 }
4196 return 0;
4197}
4198
4199unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4200 if (RetVT.SimpleTy != MVT::v4f32)
4201 return 0;
4202 if ((Subtarget->isNeonAvailable())) {
4203 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
4204 }
4205 return 0;
4206}
4207
4208unsigned fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4209 if (RetVT.SimpleTy != MVT::v2f64)
4210 return 0;
4211 if ((Subtarget->isNeonAvailable())) {
4212 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
4213 }
4214 return 0;
4215}
4216
4217unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
4218 switch (VT.SimpleTy) {
4219 case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0);
4220 case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0);
4221 case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0);
4222 case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0);
4223 case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0);
4224 case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0);
4225 case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0);
4226 case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0);
4227 default: return 0;
4228 }
4229}
4230
4231// FastEmit functions for ISD::FROUND.
4232
4233unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
4234 if (RetVT.SimpleTy != MVT::f16)
4235 return 0;
4236 if ((Subtarget->hasFullFP16())) {
4237 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
4238 }
4239 return 0;
4240}
4241
4242unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
4243 if (RetVT.SimpleTy != MVT::f32)
4244 return 0;
4245 if ((Subtarget->hasFPARMv8())) {
4246 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
4247 }
4248 return 0;
4249}
4250
4251unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
4252 if (RetVT.SimpleTy != MVT::f64)
4253 return 0;
4254 if ((Subtarget->hasFPARMv8())) {
4255 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
4256 }
4257 return 0;
4258}
4259
4260unsigned fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4261 if (RetVT.SimpleTy != MVT::v4f16)
4262 return 0;
4263 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4264 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
4265 }
4266 return 0;
4267}
4268
4269unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4270 if (RetVT.SimpleTy != MVT::v8f16)
4271 return 0;
4272 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4273 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
4274 }
4275 return 0;
4276}
4277
4278unsigned fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4279 if (RetVT.SimpleTy != MVT::v2f32)
4280 return 0;
4281 if ((Subtarget->isNeonAvailable())) {
4282 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
4283 }
4284 return 0;
4285}
4286
4287unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4288 if (RetVT.SimpleTy != MVT::v4f32)
4289 return 0;
4290 if ((Subtarget->isNeonAvailable())) {
4291 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
4292 }
4293 return 0;
4294}
4295
4296unsigned fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4297 if (RetVT.SimpleTy != MVT::v2f64)
4298 return 0;
4299 if ((Subtarget->isNeonAvailable())) {
4300 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
4301 }
4302 return 0;
4303}
4304
4305unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
4306 switch (VT.SimpleTy) {
4307 case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0);
4308 case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0);
4309 case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0);
4310 case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0);
4311 case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0);
4312 case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0);
4313 case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0);
4314 case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0);
4315 default: return 0;
4316 }
4317}
4318
4319// FastEmit functions for ISD::FROUNDEVEN.
4320
4321unsigned fastEmit_ISD_FROUNDEVEN_MVT_f16_r(MVT RetVT, unsigned Op0) {
4322 if (RetVT.SimpleTy != MVT::f16)
4323 return 0;
4324 if ((Subtarget->hasFullFP16())) {
4325 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
4326 }
4327 return 0;
4328}
4329
4330unsigned fastEmit_ISD_FROUNDEVEN_MVT_f32_r(MVT RetVT, unsigned Op0) {
4331 if (RetVT.SimpleTy != MVT::f32)
4332 return 0;
4333 if ((Subtarget->hasFPARMv8())) {
4334 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
4335 }
4336 return 0;
4337}
4338
4339unsigned fastEmit_ISD_FROUNDEVEN_MVT_f64_r(MVT RetVT, unsigned Op0) {
4340 if (RetVT.SimpleTy != MVT::f64)
4341 return 0;
4342 if ((Subtarget->hasFPARMv8())) {
4343 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
4344 }
4345 return 0;
4346}
4347
4348unsigned fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4349 if (RetVT.SimpleTy != MVT::v4f16)
4350 return 0;
4351 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4352 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
4353 }
4354 return 0;
4355}
4356
4357unsigned fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4358 if (RetVT.SimpleTy != MVT::v8f16)
4359 return 0;
4360 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4361 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
4362 }
4363 return 0;
4364}
4365
4366unsigned fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4367 if (RetVT.SimpleTy != MVT::v2f32)
4368 return 0;
4369 if ((Subtarget->isNeonAvailable())) {
4370 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
4371 }
4372 return 0;
4373}
4374
4375unsigned fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4376 if (RetVT.SimpleTy != MVT::v4f32)
4377 return 0;
4378 if ((Subtarget->isNeonAvailable())) {
4379 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
4380 }
4381 return 0;
4382}
4383
4384unsigned fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4385 if (RetVT.SimpleTy != MVT::v2f64)
4386 return 0;
4387 if ((Subtarget->isNeonAvailable())) {
4388 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
4389 }
4390 return 0;
4391}
4392
4393unsigned fastEmit_ISD_FROUNDEVEN_r(MVT VT, MVT RetVT, unsigned Op0) {
4394 switch (VT.SimpleTy) {
4395 case MVT::f16: return fastEmit_ISD_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
4396 case MVT::f32: return fastEmit_ISD_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
4397 case MVT::f64: return fastEmit_ISD_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
4398 case MVT::v4f16: return fastEmit_ISD_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
4399 case MVT::v8f16: return fastEmit_ISD_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
4400 case MVT::v2f32: return fastEmit_ISD_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
4401 case MVT::v4f32: return fastEmit_ISD_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
4402 case MVT::v2f64: return fastEmit_ISD_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
4403 default: return 0;
4404 }
4405}
4406
4407// FastEmit functions for ISD::FSQRT.
4408
4409unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0) {
4410 if (RetVT.SimpleTy != MVT::f16)
4411 return 0;
4412 if ((Subtarget->hasFullFP16())) {
4413 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
4414 }
4415 return 0;
4416}
4417
4418unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
4419 if (RetVT.SimpleTy != MVT::f32)
4420 return 0;
4421 if ((Subtarget->hasFPARMv8())) {
4422 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
4423 }
4424 return 0;
4425}
4426
4427unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
4428 if (RetVT.SimpleTy != MVT::f64)
4429 return 0;
4430 if ((Subtarget->hasFPARMv8())) {
4431 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
4432 }
4433 return 0;
4434}
4435
4436unsigned fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4437 if (RetVT.SimpleTy != MVT::v4f16)
4438 return 0;
4439 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4440 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
4441 }
4442 return 0;
4443}
4444
4445unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4446 if (RetVT.SimpleTy != MVT::v8f16)
4447 return 0;
4448 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4449 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
4450 }
4451 return 0;
4452}
4453
4454unsigned fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4455 if (RetVT.SimpleTy != MVT::v2f32)
4456 return 0;
4457 if ((Subtarget->isNeonAvailable())) {
4458 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
4459 }
4460 return 0;
4461}
4462
4463unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4464 if (RetVT.SimpleTy != MVT::v4f32)
4465 return 0;
4466 if ((Subtarget->isNeonAvailable())) {
4467 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
4468 }
4469 return 0;
4470}
4471
4472unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4473 if (RetVT.SimpleTy != MVT::v2f64)
4474 return 0;
4475 if ((Subtarget->isNeonAvailable())) {
4476 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
4477 }
4478 return 0;
4479}
4480
4481unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
4482 switch (VT.SimpleTy) {
4483 case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0);
4484 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0);
4485 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0);
4486 case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0);
4487 case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0);
4488 case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0);
4489 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0);
4490 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0);
4491 default: return 0;
4492 }
4493}
4494
4495// FastEmit functions for ISD::FTRUNC.
4496
4497unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0) {
4498 if (RetVT.SimpleTy != MVT::f16)
4499 return 0;
4500 if ((Subtarget->hasFullFP16())) {
4501 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
4502 }
4503 return 0;
4504}
4505
4506unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0) {
4507 if (RetVT.SimpleTy != MVT::f32)
4508 return 0;
4509 if ((Subtarget->hasFPARMv8())) {
4510 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
4511 }
4512 return 0;
4513}
4514
4515unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0) {
4516 if (RetVT.SimpleTy != MVT::f64)
4517 return 0;
4518 if ((Subtarget->hasFPARMv8())) {
4519 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
4520 }
4521 return 0;
4522}
4523
4524unsigned fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4525 if (RetVT.SimpleTy != MVT::v4f16)
4526 return 0;
4527 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4528 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
4529 }
4530 return 0;
4531}
4532
4533unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4534 if (RetVT.SimpleTy != MVT::v8f16)
4535 return 0;
4536 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4537 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
4538 }
4539 return 0;
4540}
4541
4542unsigned fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4543 if (RetVT.SimpleTy != MVT::v2f32)
4544 return 0;
4545 if ((Subtarget->isNeonAvailable())) {
4546 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
4547 }
4548 return 0;
4549}
4550
4551unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4552 if (RetVT.SimpleTy != MVT::v4f32)
4553 return 0;
4554 if ((Subtarget->isNeonAvailable())) {
4555 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
4556 }
4557 return 0;
4558}
4559
4560unsigned fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4561 if (RetVT.SimpleTy != MVT::v2f64)
4562 return 0;
4563 if ((Subtarget->isNeonAvailable())) {
4564 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
4565 }
4566 return 0;
4567}
4568
4569unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) {
4570 switch (VT.SimpleTy) {
4571 case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0);
4572 case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0);
4573 case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0);
4574 case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0);
4575 case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0);
4576 case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0);
4577 case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0);
4578 case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0);
4579 default: return 0;
4580 }
4581}
4582
4583// FastEmit functions for ISD::LLROUND.
4584
4585unsigned fastEmit_ISD_LLROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
4586 if (RetVT.SimpleTy != MVT::i64)
4587 return 0;
4588 if ((Subtarget->hasFullFP16())) {
4589 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4590 }
4591 return 0;
4592}
4593
4594unsigned fastEmit_ISD_LLROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
4595 if (RetVT.SimpleTy != MVT::i64)
4596 return 0;
4597 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4598}
4599
4600unsigned fastEmit_ISD_LLROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
4601 if (RetVT.SimpleTy != MVT::i64)
4602 return 0;
4603 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4604}
4605
4606unsigned fastEmit_ISD_LLROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
4607 switch (VT.SimpleTy) {
4608 case MVT::f16: return fastEmit_ISD_LLROUND_MVT_f16_r(RetVT, Op0);
4609 case MVT::f32: return fastEmit_ISD_LLROUND_MVT_f32_r(RetVT, Op0);
4610 case MVT::f64: return fastEmit_ISD_LLROUND_MVT_f64_r(RetVT, Op0);
4611 default: return 0;
4612 }
4613}
4614
4615// FastEmit functions for ISD::LROUND.
4616
4617unsigned fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(unsigned Op0) {
4618 if ((Subtarget->hasFullFP16())) {
4619 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
4620 }
4621 return 0;
4622}
4623
4624unsigned fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(unsigned Op0) {
4625 if ((Subtarget->hasFullFP16())) {
4626 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
4627 }
4628 return 0;
4629}
4630
4631unsigned fastEmit_ISD_LROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
4632switch (RetVT.SimpleTy) {
4633 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f16_MVT_i32_r(Op0);
4634 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f16_MVT_i64_r(Op0);
4635 default: return 0;
4636}
4637}
4638
4639unsigned fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(unsigned Op0) {
4640 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
4641}
4642
4643unsigned fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(unsigned Op0) {
4644 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
4645}
4646
4647unsigned fastEmit_ISD_LROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
4648switch (RetVT.SimpleTy) {
4649 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f32_MVT_i32_r(Op0);
4650 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f32_MVT_i64_r(Op0);
4651 default: return 0;
4652}
4653}
4654
4655unsigned fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(unsigned Op0) {
4656 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
4657}
4658
4659unsigned fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(unsigned Op0) {
4660 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
4661}
4662
4663unsigned fastEmit_ISD_LROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
4664switch (RetVT.SimpleTy) {
4665 case MVT::i32: return fastEmit_ISD_LROUND_MVT_f64_MVT_i32_r(Op0);
4666 case MVT::i64: return fastEmit_ISD_LROUND_MVT_f64_MVT_i64_r(Op0);
4667 default: return 0;
4668}
4669}
4670
4671unsigned fastEmit_ISD_LROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
4672 switch (VT.SimpleTy) {
4673 case MVT::f16: return fastEmit_ISD_LROUND_MVT_f16_r(RetVT, Op0);
4674 case MVT::f32: return fastEmit_ISD_LROUND_MVT_f32_r(RetVT, Op0);
4675 case MVT::f64: return fastEmit_ISD_LROUND_MVT_f64_r(RetVT, Op0);
4676 default: return 0;
4677 }
4678}
4679
4680// FastEmit functions for ISD::SINT_TO_FP.
4681
4682unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) {
4683 if ((Subtarget->hasFullFP16())) {
4684 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
4685 }
4686 return 0;
4687}
4688
4689unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
4690 if ((Subtarget->hasFPARMv8())) {
4691 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
4692 }
4693 return 0;
4694}
4695
4696unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
4697 if ((Subtarget->hasFPARMv8())) {
4698 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
4699 }
4700 return 0;
4701}
4702
4703unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
4704switch (RetVT.SimpleTy) {
4705 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
4706 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
4707 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
4708 default: return 0;
4709}
4710}
4711
4712unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) {
4713 if ((Subtarget->hasFullFP16())) {
4714 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
4715 }
4716 return 0;
4717}
4718
4719unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
4720 if ((Subtarget->hasFPARMv8())) {
4721 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
4722 }
4723 return 0;
4724}
4725
4726unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
4727 if ((Subtarget->hasFPARMv8())) {
4728 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
4729 }
4730 return 0;
4731}
4732
4733unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
4734switch (RetVT.SimpleTy) {
4735 case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
4736 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
4737 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
4738 default: return 0;
4739}
4740}
4741
4742unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
4743 if (RetVT.SimpleTy != MVT::v4f16)
4744 return 0;
4745 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4746 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
4747 }
4748 return 0;
4749}
4750
4751unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
4752 if (RetVT.SimpleTy != MVT::v8f16)
4753 return 0;
4754 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4755 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
4756 }
4757 return 0;
4758}
4759
4760unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
4761 if (RetVT.SimpleTy != MVT::v2f32)
4762 return 0;
4763 if ((Subtarget->isNeonAvailable())) {
4764 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
4765 }
4766 return 0;
4767}
4768
4769unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
4770 if (RetVT.SimpleTy != MVT::v4f32)
4771 return 0;
4772 if ((Subtarget->isNeonAvailable())) {
4773 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
4774 }
4775 return 0;
4776}
4777
4778unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
4779 if (RetVT.SimpleTy != MVT::v2f64)
4780 return 0;
4781 if ((Subtarget->isNeonAvailable())) {
4782 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
4783 }
4784 return 0;
4785}
4786
4787unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
4788 switch (VT.SimpleTy) {
4789 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
4790 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
4791 case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
4792 case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
4793 case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
4794 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
4795 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
4796 default: return 0;
4797 }
4798}
4799
4800// FastEmit functions for ISD::SPLAT_VECTOR.
4801
4802unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(unsigned Op0) {
4803 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
4804 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_B, RC: &AArch64::ZPRRegClass, Op0);
4805 }
4806 return 0;
4807}
4808
4809unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(unsigned Op0) {
4810 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
4811 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_H, RC: &AArch64::ZPRRegClass, Op0);
4812 }
4813 return 0;
4814}
4815
4816unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(unsigned Op0) {
4817 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
4818 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_S, RC: &AArch64::ZPRRegClass, Op0);
4819 }
4820 return 0;
4821}
4822
4823unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0) {
4824switch (RetVT.SimpleTy) {
4825 case MVT::nxv16i8: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv16i8_r(Op0);
4826 case MVT::nxv8i16: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv8i16_r(Op0);
4827 case MVT::nxv4i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_MVT_nxv4i32_r(Op0);
4828 default: return 0;
4829}
4830}
4831
4832unsigned fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0) {
4833 if (RetVT.SimpleTy != MVT::nxv2i64)
4834 return 0;
4835 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
4836 return fastEmitInst_r(MachineInstOpcode: AArch64::DUP_ZR_D, RC: &AArch64::ZPRRegClass, Op0);
4837 }
4838 return 0;
4839}
4840
4841unsigned fastEmit_ISD_SPLAT_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0) {
4842 switch (VT.SimpleTy) {
4843 case MVT::i32: return fastEmit_ISD_SPLAT_VECTOR_MVT_i32_r(RetVT, Op0);
4844 case MVT::i64: return fastEmit_ISD_SPLAT_VECTOR_MVT_i64_r(RetVT, Op0);
4845 default: return 0;
4846 }
4847}
4848
4849// FastEmit functions for ISD::STRICT_FCEIL.
4850
4851unsigned fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0) {
4852 if (RetVT.SimpleTy != MVT::f16)
4853 return 0;
4854 if ((Subtarget->hasFullFP16())) {
4855 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPHr, RC: &AArch64::FPR16RegClass, Op0);
4856 }
4857 return 0;
4858}
4859
4860unsigned fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0) {
4861 if (RetVT.SimpleTy != MVT::f32)
4862 return 0;
4863 if ((Subtarget->hasFPARMv8())) {
4864 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPSr, RC: &AArch64::FPR32RegClass, Op0);
4865 }
4866 return 0;
4867}
4868
4869unsigned fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0) {
4870 if (RetVT.SimpleTy != MVT::f64)
4871 return 0;
4872 if ((Subtarget->hasFPARMv8())) {
4873 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPDr, RC: &AArch64::FPR64RegClass, Op0);
4874 }
4875 return 0;
4876}
4877
4878unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4879 if (RetVT.SimpleTy != MVT::v4f16)
4880 return 0;
4881 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4882 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f16, RC: &AArch64::FPR64RegClass, Op0);
4883 }
4884 return 0;
4885}
4886
4887unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4888 if (RetVT.SimpleTy != MVT::v8f16)
4889 return 0;
4890 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4891 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv8f16, RC: &AArch64::FPR128RegClass, Op0);
4892 }
4893 return 0;
4894}
4895
4896unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4897 if (RetVT.SimpleTy != MVT::v2f32)
4898 return 0;
4899 if ((Subtarget->isNeonAvailable())) {
4900 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f32, RC: &AArch64::FPR64RegClass, Op0);
4901 }
4902 return 0;
4903}
4904
4905unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4906 if (RetVT.SimpleTy != MVT::v4f32)
4907 return 0;
4908 if ((Subtarget->isNeonAvailable())) {
4909 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv4f32, RC: &AArch64::FPR128RegClass, Op0);
4910 }
4911 return 0;
4912}
4913
4914unsigned fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
4915 if (RetVT.SimpleTy != MVT::v2f64)
4916 return 0;
4917 if ((Subtarget->isNeonAvailable())) {
4918 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTPv2f64, RC: &AArch64::FPR128RegClass, Op0);
4919 }
4920 return 0;
4921}
4922
4923unsigned fastEmit_ISD_STRICT_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0) {
4924 switch (VT.SimpleTy) {
4925 case MVT::f16: return fastEmit_ISD_STRICT_FCEIL_MVT_f16_r(RetVT, Op0);
4926 case MVT::f32: return fastEmit_ISD_STRICT_FCEIL_MVT_f32_r(RetVT, Op0);
4927 case MVT::f64: return fastEmit_ISD_STRICT_FCEIL_MVT_f64_r(RetVT, Op0);
4928 case MVT::v4f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f16_r(RetVT, Op0);
4929 case MVT::v8f16: return fastEmit_ISD_STRICT_FCEIL_MVT_v8f16_r(RetVT, Op0);
4930 case MVT::v2f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f32_r(RetVT, Op0);
4931 case MVT::v4f32: return fastEmit_ISD_STRICT_FCEIL_MVT_v4f32_r(RetVT, Op0);
4932 case MVT::v2f64: return fastEmit_ISD_STRICT_FCEIL_MVT_v2f64_r(RetVT, Op0);
4933 default: return 0;
4934 }
4935}
4936
4937// FastEmit functions for ISD::STRICT_FFLOOR.
4938
4939unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0) {
4940 if (RetVT.SimpleTy != MVT::f16)
4941 return 0;
4942 if ((Subtarget->hasFullFP16())) {
4943 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMHr, RC: &AArch64::FPR16RegClass, Op0);
4944 }
4945 return 0;
4946}
4947
4948unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0) {
4949 if (RetVT.SimpleTy != MVT::f32)
4950 return 0;
4951 if ((Subtarget->hasFPARMv8())) {
4952 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMSr, RC: &AArch64::FPR32RegClass, Op0);
4953 }
4954 return 0;
4955}
4956
4957unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0) {
4958 if (RetVT.SimpleTy != MVT::f64)
4959 return 0;
4960 if ((Subtarget->hasFPARMv8())) {
4961 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMDr, RC: &AArch64::FPR64RegClass, Op0);
4962 }
4963 return 0;
4964}
4965
4966unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
4967 if (RetVT.SimpleTy != MVT::v4f16)
4968 return 0;
4969 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4970 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f16, RC: &AArch64::FPR64RegClass, Op0);
4971 }
4972 return 0;
4973}
4974
4975unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
4976 if (RetVT.SimpleTy != MVT::v8f16)
4977 return 0;
4978 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
4979 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv8f16, RC: &AArch64::FPR128RegClass, Op0);
4980 }
4981 return 0;
4982}
4983
4984unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
4985 if (RetVT.SimpleTy != MVT::v2f32)
4986 return 0;
4987 if ((Subtarget->isNeonAvailable())) {
4988 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f32, RC: &AArch64::FPR64RegClass, Op0);
4989 }
4990 return 0;
4991}
4992
4993unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
4994 if (RetVT.SimpleTy != MVT::v4f32)
4995 return 0;
4996 if ((Subtarget->isNeonAvailable())) {
4997 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv4f32, RC: &AArch64::FPR128RegClass, Op0);
4998 }
4999 return 0;
5000}
5001
5002unsigned fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5003 if (RetVT.SimpleTy != MVT::v2f64)
5004 return 0;
5005 if ((Subtarget->isNeonAvailable())) {
5006 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTMv2f64, RC: &AArch64::FPR128RegClass, Op0);
5007 }
5008 return 0;
5009}
5010
5011unsigned fastEmit_ISD_STRICT_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0) {
5012 switch (VT.SimpleTy) {
5013 case MVT::f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_f16_r(RetVT, Op0);
5014 case MVT::f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_f32_r(RetVT, Op0);
5015 case MVT::f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_f64_r(RetVT, Op0);
5016 case MVT::v4f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f16_r(RetVT, Op0);
5017 case MVT::v8f16: return fastEmit_ISD_STRICT_FFLOOR_MVT_v8f16_r(RetVT, Op0);
5018 case MVT::v2f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f32_r(RetVT, Op0);
5019 case MVT::v4f32: return fastEmit_ISD_STRICT_FFLOOR_MVT_v4f32_r(RetVT, Op0);
5020 case MVT::v2f64: return fastEmit_ISD_STRICT_FFLOOR_MVT_v2f64_r(RetVT, Op0);
5021 default: return 0;
5022 }
5023}
5024
5025// FastEmit functions for ISD::STRICT_FNEARBYINT.
5026
5027unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
5028 if (RetVT.SimpleTy != MVT::f16)
5029 return 0;
5030 if ((Subtarget->hasFullFP16())) {
5031 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIHr, RC: &AArch64::FPR16RegClass, Op0);
5032 }
5033 return 0;
5034}
5035
5036unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5037 if (RetVT.SimpleTy != MVT::f32)
5038 return 0;
5039 if ((Subtarget->hasFPARMv8())) {
5040 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTISr, RC: &AArch64::FPR32RegClass, Op0);
5041 }
5042 return 0;
5043}
5044
5045unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
5046 if (RetVT.SimpleTy != MVT::f64)
5047 return 0;
5048 if ((Subtarget->hasFPARMv8())) {
5049 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIDr, RC: &AArch64::FPR64RegClass, Op0);
5050 }
5051 return 0;
5052}
5053
5054unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5055 if (RetVT.SimpleTy != MVT::v4f16)
5056 return 0;
5057 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5058 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f16, RC: &AArch64::FPR64RegClass, Op0);
5059 }
5060 return 0;
5061}
5062
5063unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5064 if (RetVT.SimpleTy != MVT::v8f16)
5065 return 0;
5066 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5067 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv8f16, RC: &AArch64::FPR128RegClass, Op0);
5068 }
5069 return 0;
5070}
5071
5072unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5073 if (RetVT.SimpleTy != MVT::v2f32)
5074 return 0;
5075 if ((Subtarget->isNeonAvailable())) {
5076 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f32, RC: &AArch64::FPR64RegClass, Op0);
5077 }
5078 return 0;
5079}
5080
5081unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5082 if (RetVT.SimpleTy != MVT::v4f32)
5083 return 0;
5084 if ((Subtarget->isNeonAvailable())) {
5085 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv4f32, RC: &AArch64::FPR128RegClass, Op0);
5086 }
5087 return 0;
5088}
5089
5090unsigned fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5091 if (RetVT.SimpleTy != MVT::v2f64)
5092 return 0;
5093 if ((Subtarget->isNeonAvailable())) {
5094 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTIv2f64, RC: &AArch64::FPR128RegClass, Op0);
5095 }
5096 return 0;
5097}
5098
5099unsigned fastEmit_ISD_STRICT_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0) {
5100 switch (VT.SimpleTy) {
5101 case MVT::f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f16_r(RetVT, Op0);
5102 case MVT::f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f32_r(RetVT, Op0);
5103 case MVT::f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_f64_r(RetVT, Op0);
5104 case MVT::v4f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f16_r(RetVT, Op0);
5105 case MVT::v8f16: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v8f16_r(RetVT, Op0);
5106 case MVT::v2f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f32_r(RetVT, Op0);
5107 case MVT::v4f32: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v4f32_r(RetVT, Op0);
5108 case MVT::v2f64: return fastEmit_ISD_STRICT_FNEARBYINT_MVT_v2f64_r(RetVT, Op0);
5109 default: return 0;
5110 }
5111}
5112
5113// FastEmit functions for ISD::STRICT_FP_EXTEND.
5114
5115unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0) {
5116 if ((Subtarget->hasFPARMv8())) {
5117 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSHr, RC: &AArch64::FPR32RegClass, Op0);
5118 }
5119 return 0;
5120}
5121
5122unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0) {
5123 if ((Subtarget->hasFPARMv8())) {
5124 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDHr, RC: &AArch64::FPR64RegClass, Op0);
5125 }
5126 return 0;
5127}
5128
5129unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0) {
5130switch (RetVT.SimpleTy) {
5131 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f32_r(Op0);
5132 case MVT::f64: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_MVT_f64_r(Op0);
5133 default: return 0;
5134}
5135}
5136
5137unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
5138 if (RetVT.SimpleTy != MVT::f64)
5139 return 0;
5140 if ((Subtarget->hasFPARMv8())) {
5141 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTDSr, RC: &AArch64::FPR64RegClass, Op0);
5142 }
5143 return 0;
5144}
5145
5146unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5147 if (RetVT.SimpleTy != MVT::v4f32)
5148 return 0;
5149 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv4i16, RC: &AArch64::FPR128RegClass, Op0);
5150}
5151
5152unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
5153 if (RetVT.SimpleTy != MVT::v4f32)
5154 return 0;
5155 return fastEmitInst_r(MachineInstOpcode: AArch64::SHLLv4i16, RC: &AArch64::FPR128RegClass, Op0);
5156}
5157
5158unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5159 if (RetVT.SimpleTy != MVT::v2f64)
5160 return 0;
5161 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTLv2i32, RC: &AArch64::FPR128RegClass, Op0);
5162}
5163
5164unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
5165 switch (VT.SimpleTy) {
5166 case MVT::f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f16_r(RetVT, Op0);
5167 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0);
5168 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f16_r(RetVT, Op0);
5169 case MVT::v4bf16: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4bf16_r(RetVT, Op0);
5170 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v2f32_r(RetVT, Op0);
5171 default: return 0;
5172 }
5173}
5174
5175// FastEmit functions for ISD::STRICT_FP_ROUND.
5176
5177unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(unsigned Op0) {
5178 if ((Subtarget->hasBF16()) && (Subtarget->hasNEON())) {
5179 return fastEmitInst_r(MachineInstOpcode: AArch64::BFCVT, RC: &AArch64::FPR16RegClass, Op0);
5180 }
5181 return 0;
5182}
5183
5184unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(unsigned Op0) {
5185 if ((Subtarget->hasFPARMv8())) {
5186 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHSr, RC: &AArch64::FPR16RegClass, Op0);
5187 }
5188 return 0;
5189}
5190
5191unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
5192switch (RetVT.SimpleTy) {
5193 case MVT::bf16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_bf16_r(Op0);
5194 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_MVT_f16_r(Op0);
5195 default: return 0;
5196}
5197}
5198
5199unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0) {
5200 if ((Subtarget->hasFPARMv8())) {
5201 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTHDr, RC: &AArch64::FPR16RegClass, Op0);
5202 }
5203 return 0;
5204}
5205
5206unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0) {
5207 if ((Subtarget->hasFPARMv8())) {
5208 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTSDr, RC: &AArch64::FPR32RegClass, Op0);
5209 }
5210 return 0;
5211}
5212
5213unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
5214switch (RetVT.SimpleTy) {
5215 case MVT::f16: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f16_r(Op0);
5216 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_MVT_f32_r(Op0);
5217 default: return 0;
5218}
5219}
5220
5221unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5222 if (RetVT.SimpleTy != MVT::v4f16)
5223 return 0;
5224 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
5225}
5226
5227unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5228 if (RetVT.SimpleTy != MVT::v2f32)
5229 return 0;
5230 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
5231}
5232
5233unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
5234 switch (VT.SimpleTy) {
5235 case MVT::f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f32_r(RetVT, Op0);
5236 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0);
5237 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v4f32_r(RetVT, Op0);
5238 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_v2f64_r(RetVT, Op0);
5239 default: return 0;
5240 }
5241}
5242
5243// FastEmit functions for ISD::STRICT_FP_TO_SINT.
5244
5245unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0) {
5246 if ((Subtarget->hasFullFP16())) {
5247 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWHr, RC: &AArch64::GPR32RegClass, Op0);
5248 }
5249 return 0;
5250}
5251
5252unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0) {
5253 if ((Subtarget->hasFullFP16())) {
5254 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXHr, RC: &AArch64::GPR64RegClass, Op0);
5255 }
5256 return 0;
5257}
5258
5259unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
5260switch (RetVT.SimpleTy) {
5261 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0);
5262 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0);
5263 default: return 0;
5264}
5265}
5266
5267unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0) {
5268 if ((Subtarget->hasFPARMv8())) {
5269 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWSr, RC: &AArch64::GPR32RegClass, Op0);
5270 }
5271 return 0;
5272}
5273
5274unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0) {
5275 if ((Subtarget->hasFPARMv8())) {
5276 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXSr, RC: &AArch64::GPR64RegClass, Op0);
5277 }
5278 return 0;
5279}
5280
5281unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5282switch (RetVT.SimpleTy) {
5283 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0);
5284 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0);
5285 default: return 0;
5286}
5287}
5288
5289unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0) {
5290 if ((Subtarget->hasFPARMv8())) {
5291 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUWDr, RC: &AArch64::GPR32RegClass, Op0);
5292 }
5293 return 0;
5294}
5295
5296unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0) {
5297 if ((Subtarget->hasFPARMv8())) {
5298 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSUXDr, RC: &AArch64::GPR64RegClass, Op0);
5299 }
5300 return 0;
5301}
5302
5303unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
5304switch (RetVT.SimpleTy) {
5305 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0);
5306 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0);
5307 default: return 0;
5308}
5309}
5310
5311unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5312 if (RetVT.SimpleTy != MVT::v4i16)
5313 return 0;
5314 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5315 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f16, RC: &AArch64::FPR64RegClass, Op0);
5316 }
5317 return 0;
5318}
5319
5320unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5321 if (RetVT.SimpleTy != MVT::v8i16)
5322 return 0;
5323 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5324 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv8f16, RC: &AArch64::FPR128RegClass, Op0);
5325 }
5326 return 0;
5327}
5328
5329unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5330 if (RetVT.SimpleTy != MVT::v2i32)
5331 return 0;
5332 if ((Subtarget->isNeonAvailable())) {
5333 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f32, RC: &AArch64::FPR64RegClass, Op0);
5334 }
5335 return 0;
5336}
5337
5338unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5339 if (RetVT.SimpleTy != MVT::v4i32)
5340 return 0;
5341 if ((Subtarget->isNeonAvailable())) {
5342 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv4f32, RC: &AArch64::FPR128RegClass, Op0);
5343 }
5344 return 0;
5345}
5346
5347unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5348 if (RetVT.SimpleTy != MVT::v2i64)
5349 return 0;
5350 if ((Subtarget->isNeonAvailable())) {
5351 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZSv2f64, RC: &AArch64::FPR128RegClass, Op0);
5352 }
5353 return 0;
5354}
5355
5356unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0) {
5357 switch (VT.SimpleTy) {
5358 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f16_r(RetVT, Op0);
5359 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0);
5360 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0);
5361 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0);
5362 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0);
5363 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0);
5364 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0);
5365 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0);
5366 default: return 0;
5367 }
5368}
5369
5370// FastEmit functions for ISD::STRICT_FP_TO_UINT.
5371
5372unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0) {
5373 if ((Subtarget->hasFullFP16())) {
5374 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWHr, RC: &AArch64::GPR32RegClass, Op0);
5375 }
5376 return 0;
5377}
5378
5379unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0) {
5380 if ((Subtarget->hasFullFP16())) {
5381 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXHr, RC: &AArch64::GPR64RegClass, Op0);
5382 }
5383 return 0;
5384}
5385
5386unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
5387switch (RetVT.SimpleTy) {
5388 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0);
5389 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0);
5390 default: return 0;
5391}
5392}
5393
5394unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0) {
5395 if ((Subtarget->hasFPARMv8())) {
5396 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWSr, RC: &AArch64::GPR32RegClass, Op0);
5397 }
5398 return 0;
5399}
5400
5401unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0) {
5402 if ((Subtarget->hasFPARMv8())) {
5403 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXSr, RC: &AArch64::GPR64RegClass, Op0);
5404 }
5405 return 0;
5406}
5407
5408unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5409switch (RetVT.SimpleTy) {
5410 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0);
5411 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0);
5412 default: return 0;
5413}
5414}
5415
5416unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0) {
5417 if ((Subtarget->hasFPARMv8())) {
5418 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUWDr, RC: &AArch64::GPR32RegClass, Op0);
5419 }
5420 return 0;
5421}
5422
5423unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0) {
5424 if ((Subtarget->hasFPARMv8())) {
5425 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUUXDr, RC: &AArch64::GPR64RegClass, Op0);
5426 }
5427 return 0;
5428}
5429
5430unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
5431switch (RetVT.SimpleTy) {
5432 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0);
5433 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0);
5434 default: return 0;
5435}
5436}
5437
5438unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5439 if (RetVT.SimpleTy != MVT::v4i16)
5440 return 0;
5441 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5442 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f16, RC: &AArch64::FPR64RegClass, Op0);
5443 }
5444 return 0;
5445}
5446
5447unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5448 if (RetVT.SimpleTy != MVT::v8i16)
5449 return 0;
5450 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5451 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv8f16, RC: &AArch64::FPR128RegClass, Op0);
5452 }
5453 return 0;
5454}
5455
5456unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5457 if (RetVT.SimpleTy != MVT::v2i32)
5458 return 0;
5459 if ((Subtarget->isNeonAvailable())) {
5460 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f32, RC: &AArch64::FPR64RegClass, Op0);
5461 }
5462 return 0;
5463}
5464
5465unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5466 if (RetVT.SimpleTy != MVT::v4i32)
5467 return 0;
5468 if ((Subtarget->isNeonAvailable())) {
5469 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv4f32, RC: &AArch64::FPR128RegClass, Op0);
5470 }
5471 return 0;
5472}
5473
5474unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5475 if (RetVT.SimpleTy != MVT::v2i64)
5476 return 0;
5477 if ((Subtarget->isNeonAvailable())) {
5478 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTZUv2f64, RC: &AArch64::FPR128RegClass, Op0);
5479 }
5480 return 0;
5481}
5482
5483unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0) {
5484 switch (VT.SimpleTy) {
5485 case MVT::f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f16_r(RetVT, Op0);
5486 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0);
5487 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0);
5488 case MVT::v4f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0);
5489 case MVT::v8f16: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0);
5490 case MVT::v2f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0);
5491 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0);
5492 case MVT::v2f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0);
5493 default: return 0;
5494 }
5495}
5496
5497// FastEmit functions for ISD::STRICT_FRINT.
5498
5499unsigned fastEmit_ISD_STRICT_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
5500 if (RetVT.SimpleTy != MVT::f16)
5501 return 0;
5502 if ((Subtarget->hasFullFP16())) {
5503 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXHr, RC: &AArch64::FPR16RegClass, Op0);
5504 }
5505 return 0;
5506}
5507
5508unsigned fastEmit_ISD_STRICT_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5509 if (RetVT.SimpleTy != MVT::f32)
5510 return 0;
5511 if ((Subtarget->hasFPARMv8())) {
5512 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXSr, RC: &AArch64::FPR32RegClass, Op0);
5513 }
5514 return 0;
5515}
5516
5517unsigned fastEmit_ISD_STRICT_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
5518 if (RetVT.SimpleTy != MVT::f64)
5519 return 0;
5520 if ((Subtarget->hasFPARMv8())) {
5521 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXDr, RC: &AArch64::FPR64RegClass, Op0);
5522 }
5523 return 0;
5524}
5525
5526unsigned fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5527 if (RetVT.SimpleTy != MVT::v4f16)
5528 return 0;
5529 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5530 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f16, RC: &AArch64::FPR64RegClass, Op0);
5531 }
5532 return 0;
5533}
5534
5535unsigned fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5536 if (RetVT.SimpleTy != MVT::v8f16)
5537 return 0;
5538 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5539 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv8f16, RC: &AArch64::FPR128RegClass, Op0);
5540 }
5541 return 0;
5542}
5543
5544unsigned fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5545 if (RetVT.SimpleTy != MVT::v2f32)
5546 return 0;
5547 if ((Subtarget->isNeonAvailable())) {
5548 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f32, RC: &AArch64::FPR64RegClass, Op0);
5549 }
5550 return 0;
5551}
5552
5553unsigned fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5554 if (RetVT.SimpleTy != MVT::v4f32)
5555 return 0;
5556 if ((Subtarget->isNeonAvailable())) {
5557 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv4f32, RC: &AArch64::FPR128RegClass, Op0);
5558 }
5559 return 0;
5560}
5561
5562unsigned fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5563 if (RetVT.SimpleTy != MVT::v2f64)
5564 return 0;
5565 if ((Subtarget->isNeonAvailable())) {
5566 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTXv2f64, RC: &AArch64::FPR128RegClass, Op0);
5567 }
5568 return 0;
5569}
5570
5571unsigned fastEmit_ISD_STRICT_FRINT_r(MVT VT, MVT RetVT, unsigned Op0) {
5572 switch (VT.SimpleTy) {
5573 case MVT::f16: return fastEmit_ISD_STRICT_FRINT_MVT_f16_r(RetVT, Op0);
5574 case MVT::f32: return fastEmit_ISD_STRICT_FRINT_MVT_f32_r(RetVT, Op0);
5575 case MVT::f64: return fastEmit_ISD_STRICT_FRINT_MVT_f64_r(RetVT, Op0);
5576 case MVT::v4f16: return fastEmit_ISD_STRICT_FRINT_MVT_v4f16_r(RetVT, Op0);
5577 case MVT::v8f16: return fastEmit_ISD_STRICT_FRINT_MVT_v8f16_r(RetVT, Op0);
5578 case MVT::v2f32: return fastEmit_ISD_STRICT_FRINT_MVT_v2f32_r(RetVT, Op0);
5579 case MVT::v4f32: return fastEmit_ISD_STRICT_FRINT_MVT_v4f32_r(RetVT, Op0);
5580 case MVT::v2f64: return fastEmit_ISD_STRICT_FRINT_MVT_v2f64_r(RetVT, Op0);
5581 default: return 0;
5582 }
5583}
5584
5585// FastEmit functions for ISD::STRICT_FROUND.
5586
5587unsigned fastEmit_ISD_STRICT_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
5588 if (RetVT.SimpleTy != MVT::f16)
5589 return 0;
5590 if ((Subtarget->hasFullFP16())) {
5591 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAHr, RC: &AArch64::FPR16RegClass, Op0);
5592 }
5593 return 0;
5594}
5595
5596unsigned fastEmit_ISD_STRICT_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
5597 if (RetVT.SimpleTy != MVT::f32)
5598 return 0;
5599 if ((Subtarget->hasFPARMv8())) {
5600 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTASr, RC: &AArch64::FPR32RegClass, Op0);
5601 }
5602 return 0;
5603}
5604
5605unsigned fastEmit_ISD_STRICT_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
5606 if (RetVT.SimpleTy != MVT::f64)
5607 return 0;
5608 if ((Subtarget->hasFPARMv8())) {
5609 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTADr, RC: &AArch64::FPR64RegClass, Op0);
5610 }
5611 return 0;
5612}
5613
5614unsigned fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5615 if (RetVT.SimpleTy != MVT::v4f16)
5616 return 0;
5617 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5618 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f16, RC: &AArch64::FPR64RegClass, Op0);
5619 }
5620 return 0;
5621}
5622
5623unsigned fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5624 if (RetVT.SimpleTy != MVT::v8f16)
5625 return 0;
5626 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5627 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv8f16, RC: &AArch64::FPR128RegClass, Op0);
5628 }
5629 return 0;
5630}
5631
5632unsigned fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5633 if (RetVT.SimpleTy != MVT::v2f32)
5634 return 0;
5635 if ((Subtarget->isNeonAvailable())) {
5636 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f32, RC: &AArch64::FPR64RegClass, Op0);
5637 }
5638 return 0;
5639}
5640
5641unsigned fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5642 if (RetVT.SimpleTy != MVT::v4f32)
5643 return 0;
5644 if ((Subtarget->isNeonAvailable())) {
5645 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv4f32, RC: &AArch64::FPR128RegClass, Op0);
5646 }
5647 return 0;
5648}
5649
5650unsigned fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5651 if (RetVT.SimpleTy != MVT::v2f64)
5652 return 0;
5653 if ((Subtarget->isNeonAvailable())) {
5654 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTAv2f64, RC: &AArch64::FPR128RegClass, Op0);
5655 }
5656 return 0;
5657}
5658
5659unsigned fastEmit_ISD_STRICT_FROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
5660 switch (VT.SimpleTy) {
5661 case MVT::f16: return fastEmit_ISD_STRICT_FROUND_MVT_f16_r(RetVT, Op0);
5662 case MVT::f32: return fastEmit_ISD_STRICT_FROUND_MVT_f32_r(RetVT, Op0);
5663 case MVT::f64: return fastEmit_ISD_STRICT_FROUND_MVT_f64_r(RetVT, Op0);
5664 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUND_MVT_v4f16_r(RetVT, Op0);
5665 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUND_MVT_v8f16_r(RetVT, Op0);
5666 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUND_MVT_v2f32_r(RetVT, Op0);
5667 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUND_MVT_v4f32_r(RetVT, Op0);
5668 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUND_MVT_v2f64_r(RetVT, Op0);
5669 default: return 0;
5670 }
5671}
5672
5673// FastEmit functions for ISD::STRICT_FROUNDEVEN.
5674
5675unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(MVT RetVT, unsigned Op0) {
5676 if (RetVT.SimpleTy != MVT::f16)
5677 return 0;
5678 if ((Subtarget->hasFullFP16())) {
5679 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNHr, RC: &AArch64::FPR16RegClass, Op0);
5680 }
5681 return 0;
5682}
5683
5684unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(MVT RetVT, unsigned Op0) {
5685 if (RetVT.SimpleTy != MVT::f32)
5686 return 0;
5687 if ((Subtarget->hasFPARMv8())) {
5688 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNSr, RC: &AArch64::FPR32RegClass, Op0);
5689 }
5690 return 0;
5691}
5692
5693unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(MVT RetVT, unsigned Op0) {
5694 if (RetVT.SimpleTy != MVT::f64)
5695 return 0;
5696 if ((Subtarget->hasFPARMv8())) {
5697 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNDr, RC: &AArch64::FPR64RegClass, Op0);
5698 }
5699 return 0;
5700}
5701
5702unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5703 if (RetVT.SimpleTy != MVT::v4f16)
5704 return 0;
5705 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5706 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f16, RC: &AArch64::FPR64RegClass, Op0);
5707 }
5708 return 0;
5709}
5710
5711unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5712 if (RetVT.SimpleTy != MVT::v8f16)
5713 return 0;
5714 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5715 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv8f16, RC: &AArch64::FPR128RegClass, Op0);
5716 }
5717 return 0;
5718}
5719
5720unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5721 if (RetVT.SimpleTy != MVT::v2f32)
5722 return 0;
5723 if ((Subtarget->isNeonAvailable())) {
5724 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f32, RC: &AArch64::FPR64RegClass, Op0);
5725 }
5726 return 0;
5727}
5728
5729unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5730 if (RetVT.SimpleTy != MVT::v4f32)
5731 return 0;
5732 if ((Subtarget->isNeonAvailable())) {
5733 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv4f32, RC: &AArch64::FPR128RegClass, Op0);
5734 }
5735 return 0;
5736}
5737
5738unsigned fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5739 if (RetVT.SimpleTy != MVT::v2f64)
5740 return 0;
5741 if ((Subtarget->isNeonAvailable())) {
5742 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTNv2f64, RC: &AArch64::FPR128RegClass, Op0);
5743 }
5744 return 0;
5745}
5746
5747unsigned fastEmit_ISD_STRICT_FROUNDEVEN_r(MVT VT, MVT RetVT, unsigned Op0) {
5748 switch (VT.SimpleTy) {
5749 case MVT::f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f16_r(RetVT, Op0);
5750 case MVT::f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f32_r(RetVT, Op0);
5751 case MVT::f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_f64_r(RetVT, Op0);
5752 case MVT::v4f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f16_r(RetVT, Op0);
5753 case MVT::v8f16: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v8f16_r(RetVT, Op0);
5754 case MVT::v2f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f32_r(RetVT, Op0);
5755 case MVT::v4f32: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v4f32_r(RetVT, Op0);
5756 case MVT::v2f64: return fastEmit_ISD_STRICT_FROUNDEVEN_MVT_v2f64_r(RetVT, Op0);
5757 default: return 0;
5758 }
5759}
5760
5761// FastEmit functions for ISD::STRICT_FSQRT.
5762
5763unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0) {
5764 if (RetVT.SimpleTy != MVT::f16)
5765 return 0;
5766 if ((Subtarget->hasFullFP16())) {
5767 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTHr, RC: &AArch64::FPR16RegClass, Op0);
5768 }
5769 return 0;
5770}
5771
5772unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0) {
5773 if (RetVT.SimpleTy != MVT::f32)
5774 return 0;
5775 if ((Subtarget->hasFPARMv8())) {
5776 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTSr, RC: &AArch64::FPR32RegClass, Op0);
5777 }
5778 return 0;
5779}
5780
5781unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0) {
5782 if (RetVT.SimpleTy != MVT::f64)
5783 return 0;
5784 if ((Subtarget->hasFPARMv8())) {
5785 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTDr, RC: &AArch64::FPR64RegClass, Op0);
5786 }
5787 return 0;
5788}
5789
5790unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5791 if (RetVT.SimpleTy != MVT::v4f16)
5792 return 0;
5793 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5794 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f16, RC: &AArch64::FPR64RegClass, Op0);
5795 }
5796 return 0;
5797}
5798
5799unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5800 if (RetVT.SimpleTy != MVT::v8f16)
5801 return 0;
5802 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5803 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv8f16, RC: &AArch64::FPR128RegClass, Op0);
5804 }
5805 return 0;
5806}
5807
5808unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5809 if (RetVT.SimpleTy != MVT::v2f32)
5810 return 0;
5811 if ((Subtarget->isNeonAvailable())) {
5812 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f32, RC: &AArch64::FPR64RegClass, Op0);
5813 }
5814 return 0;
5815}
5816
5817unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5818 if (RetVT.SimpleTy != MVT::v4f32)
5819 return 0;
5820 if ((Subtarget->isNeonAvailable())) {
5821 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv4f32, RC: &AArch64::FPR128RegClass, Op0);
5822 }
5823 return 0;
5824}
5825
5826unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5827 if (RetVT.SimpleTy != MVT::v2f64)
5828 return 0;
5829 if ((Subtarget->isNeonAvailable())) {
5830 return fastEmitInst_r(MachineInstOpcode: AArch64::FSQRTv2f64, RC: &AArch64::FPR128RegClass, Op0);
5831 }
5832 return 0;
5833}
5834
5835unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0) {
5836 switch (VT.SimpleTy) {
5837 case MVT::f16: return fastEmit_ISD_STRICT_FSQRT_MVT_f16_r(RetVT, Op0);
5838 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0);
5839 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0);
5840 case MVT::v4f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f16_r(RetVT, Op0);
5841 case MVT::v8f16: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f16_r(RetVT, Op0);
5842 case MVT::v2f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f32_r(RetVT, Op0);
5843 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0);
5844 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0);
5845 default: return 0;
5846 }
5847}
5848
5849// FastEmit functions for ISD::STRICT_FTRUNC.
5850
5851unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0) {
5852 if (RetVT.SimpleTy != MVT::f16)
5853 return 0;
5854 if ((Subtarget->hasFullFP16())) {
5855 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZHr, RC: &AArch64::FPR16RegClass, Op0);
5856 }
5857 return 0;
5858}
5859
5860unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0) {
5861 if (RetVT.SimpleTy != MVT::f32)
5862 return 0;
5863 if ((Subtarget->hasFPARMv8())) {
5864 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZSr, RC: &AArch64::FPR32RegClass, Op0);
5865 }
5866 return 0;
5867}
5868
5869unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0) {
5870 if (RetVT.SimpleTy != MVT::f64)
5871 return 0;
5872 if ((Subtarget->hasFPARMv8())) {
5873 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZDr, RC: &AArch64::FPR64RegClass, Op0);
5874 }
5875 return 0;
5876}
5877
5878unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
5879 if (RetVT.SimpleTy != MVT::v4f16)
5880 return 0;
5881 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5882 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f16, RC: &AArch64::FPR64RegClass, Op0);
5883 }
5884 return 0;
5885}
5886
5887unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
5888 if (RetVT.SimpleTy != MVT::v8f16)
5889 return 0;
5890 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
5891 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv8f16, RC: &AArch64::FPR128RegClass, Op0);
5892 }
5893 return 0;
5894}
5895
5896unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
5897 if (RetVT.SimpleTy != MVT::v2f32)
5898 return 0;
5899 if ((Subtarget->isNeonAvailable())) {
5900 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f32, RC: &AArch64::FPR64RegClass, Op0);
5901 }
5902 return 0;
5903}
5904
5905unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
5906 if (RetVT.SimpleTy != MVT::v4f32)
5907 return 0;
5908 if ((Subtarget->isNeonAvailable())) {
5909 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv4f32, RC: &AArch64::FPR128RegClass, Op0);
5910 }
5911 return 0;
5912}
5913
5914unsigned fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
5915 if (RetVT.SimpleTy != MVT::v2f64)
5916 return 0;
5917 if ((Subtarget->isNeonAvailable())) {
5918 return fastEmitInst_r(MachineInstOpcode: AArch64::FRINTZv2f64, RC: &AArch64::FPR128RegClass, Op0);
5919 }
5920 return 0;
5921}
5922
5923unsigned fastEmit_ISD_STRICT_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0) {
5924 switch (VT.SimpleTy) {
5925 case MVT::f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_f16_r(RetVT, Op0);
5926 case MVT::f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_f32_r(RetVT, Op0);
5927 case MVT::f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_f64_r(RetVT, Op0);
5928 case MVT::v4f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f16_r(RetVT, Op0);
5929 case MVT::v8f16: return fastEmit_ISD_STRICT_FTRUNC_MVT_v8f16_r(RetVT, Op0);
5930 case MVT::v2f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f32_r(RetVT, Op0);
5931 case MVT::v4f32: return fastEmit_ISD_STRICT_FTRUNC_MVT_v4f32_r(RetVT, Op0);
5932 case MVT::v2f64: return fastEmit_ISD_STRICT_FTRUNC_MVT_v2f64_r(RetVT, Op0);
5933 default: return 0;
5934 }
5935}
5936
5937// FastEmit functions for ISD::STRICT_LLROUND.
5938
5939unsigned fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
5940 if (RetVT.SimpleTy != MVT::i64)
5941 return 0;
5942 if ((Subtarget->hasFullFP16())) {
5943 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5944 }
5945 return 0;
5946}
5947
5948unsigned fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
5949 if (RetVT.SimpleTy != MVT::i64)
5950 return 0;
5951 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5952}
5953
5954unsigned fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
5955 if (RetVT.SimpleTy != MVT::i64)
5956 return 0;
5957 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
5958}
5959
5960unsigned fastEmit_ISD_STRICT_LLROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
5961 switch (VT.SimpleTy) {
5962 case MVT::f16: return fastEmit_ISD_STRICT_LLROUND_MVT_f16_r(RetVT, Op0);
5963 case MVT::f32: return fastEmit_ISD_STRICT_LLROUND_MVT_f32_r(RetVT, Op0);
5964 case MVT::f64: return fastEmit_ISD_STRICT_LLROUND_MVT_f64_r(RetVT, Op0);
5965 default: return 0;
5966 }
5967}
5968
5969// FastEmit functions for ISD::STRICT_LROUND.
5970
5971unsigned fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(unsigned Op0) {
5972 if ((Subtarget->hasFullFP16())) {
5973 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWHr, RC: &AArch64::GPR32RegClass, Op0);
5974 }
5975 return 0;
5976}
5977
5978unsigned fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(unsigned Op0) {
5979 if ((Subtarget->hasFullFP16())) {
5980 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXHr, RC: &AArch64::GPR64RegClass, Op0);
5981 }
5982 return 0;
5983}
5984
5985unsigned fastEmit_ISD_STRICT_LROUND_MVT_f16_r(MVT RetVT, unsigned Op0) {
5986switch (RetVT.SimpleTy) {
5987 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i32_r(Op0);
5988 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f16_MVT_i64_r(Op0);
5989 default: return 0;
5990}
5991}
5992
5993unsigned fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(unsigned Op0) {
5994 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWSr, RC: &AArch64::GPR32RegClass, Op0);
5995}
5996
5997unsigned fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(unsigned Op0) {
5998 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXSr, RC: &AArch64::GPR64RegClass, Op0);
5999}
6000
6001unsigned fastEmit_ISD_STRICT_LROUND_MVT_f32_r(MVT RetVT, unsigned Op0) {
6002switch (RetVT.SimpleTy) {
6003 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i32_r(Op0);
6004 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f32_MVT_i64_r(Op0);
6005 default: return 0;
6006}
6007}
6008
6009unsigned fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(unsigned Op0) {
6010 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUWDr, RC: &AArch64::GPR32RegClass, Op0);
6011}
6012
6013unsigned fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(unsigned Op0) {
6014 return fastEmitInst_r(MachineInstOpcode: AArch64::FCVTASUXDr, RC: &AArch64::GPR64RegClass, Op0);
6015}
6016
6017unsigned fastEmit_ISD_STRICT_LROUND_MVT_f64_r(MVT RetVT, unsigned Op0) {
6018switch (RetVT.SimpleTy) {
6019 case MVT::i32: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i32_r(Op0);
6020 case MVT::i64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_MVT_i64_r(Op0);
6021 default: return 0;
6022}
6023}
6024
6025unsigned fastEmit_ISD_STRICT_LROUND_r(MVT VT, MVT RetVT, unsigned Op0) {
6026 switch (VT.SimpleTy) {
6027 case MVT::f16: return fastEmit_ISD_STRICT_LROUND_MVT_f16_r(RetVT, Op0);
6028 case MVT::f32: return fastEmit_ISD_STRICT_LROUND_MVT_f32_r(RetVT, Op0);
6029 case MVT::f64: return fastEmit_ISD_STRICT_LROUND_MVT_f64_r(RetVT, Op0);
6030 default: return 0;
6031 }
6032}
6033
6034// FastEmit functions for ISD::STRICT_SINT_TO_FP.
6035
6036unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) {
6037 if ((Subtarget->hasFullFP16())) {
6038 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
6039 }
6040 return 0;
6041}
6042
6043unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
6044 if ((Subtarget->hasFPARMv8())) {
6045 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
6046 }
6047 return 0;
6048}
6049
6050unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
6051 if ((Subtarget->hasFPARMv8())) {
6052 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6053 }
6054 return 0;
6055}
6056
6057unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
6058switch (RetVT.SimpleTy) {
6059 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6060 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6061 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6062 default: return 0;
6063}
6064}
6065
6066unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) {
6067 if ((Subtarget->hasFullFP16())) {
6068 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6069 }
6070 return 0;
6071}
6072
6073unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
6074 if ((Subtarget->hasFPARMv8())) {
6075 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6076 }
6077 return 0;
6078}
6079
6080unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
6081 if ((Subtarget->hasFPARMv8())) {
6082 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6083 }
6084 return 0;
6085}
6086
6087unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
6088switch (RetVT.SimpleTy) {
6089 case MVT::f16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6090 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6091 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6092 default: return 0;
6093}
6094}
6095
6096unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6097 if (RetVT.SimpleTy != MVT::v4f16)
6098 return 0;
6099 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6100 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6101 }
6102 return 0;
6103}
6104
6105unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6106 if (RetVT.SimpleTy != MVT::v8f16)
6107 return 0;
6108 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6109 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6110 }
6111 return 0;
6112}
6113
6114unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
6115 if (RetVT.SimpleTy != MVT::v2f32)
6116 return 0;
6117 if ((Subtarget->isNeonAvailable())) {
6118 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6119 }
6120 return 0;
6121}
6122
6123unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6124 if (RetVT.SimpleTy != MVT::v4f32)
6125 return 0;
6126 if ((Subtarget->isNeonAvailable())) {
6127 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6128 }
6129 return 0;
6130}
6131
6132unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6133 if (RetVT.SimpleTy != MVT::v2f64)
6134 return 0;
6135 if ((Subtarget->isNeonAvailable())) {
6136 return fastEmitInst_r(MachineInstOpcode: AArch64::SCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6137 }
6138 return 0;
6139}
6140
6141unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
6142 switch (VT.SimpleTy) {
6143 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0);
6144 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0);
6145 case MVT::v4i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6146 case MVT::v8i16: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6147 case MVT::v2i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6148 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6149 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6150 default: return 0;
6151 }
6152}
6153
6154// FastEmit functions for ISD::STRICT_UINT_TO_FP.
6155
6156unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) {
6157 if ((Subtarget->hasFullFP16())) {
6158 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
6159 }
6160 return 0;
6161}
6162
6163unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
6164 if ((Subtarget->hasFPARMv8())) {
6165 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
6166 }
6167 return 0;
6168}
6169
6170unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
6171 if ((Subtarget->hasFPARMv8())) {
6172 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6173 }
6174 return 0;
6175}
6176
6177unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
6178switch (RetVT.SimpleTy) {
6179 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6180 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6181 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6182 default: return 0;
6183}
6184}
6185
6186unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) {
6187 if ((Subtarget->hasFullFP16())) {
6188 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6189 }
6190 return 0;
6191}
6192
6193unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
6194 if ((Subtarget->hasFPARMv8())) {
6195 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6196 }
6197 return 0;
6198}
6199
6200unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
6201 if ((Subtarget->hasFPARMv8())) {
6202 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6203 }
6204 return 0;
6205}
6206
6207unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
6208switch (RetVT.SimpleTy) {
6209 case MVT::f16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6210 case MVT::f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6211 case MVT::f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6212 default: return 0;
6213}
6214}
6215
6216unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6217 if (RetVT.SimpleTy != MVT::v4f16)
6218 return 0;
6219 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6220 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6221 }
6222 return 0;
6223}
6224
6225unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6226 if (RetVT.SimpleTy != MVT::v8f16)
6227 return 0;
6228 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6229 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6230 }
6231 return 0;
6232}
6233
6234unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
6235 if (RetVT.SimpleTy != MVT::v2f32)
6236 return 0;
6237 if ((Subtarget->isNeonAvailable())) {
6238 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6239 }
6240 return 0;
6241}
6242
6243unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6244 if (RetVT.SimpleTy != MVT::v4f32)
6245 return 0;
6246 if ((Subtarget->isNeonAvailable())) {
6247 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6248 }
6249 return 0;
6250}
6251
6252unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6253 if (RetVT.SimpleTy != MVT::v2f64)
6254 return 0;
6255 if ((Subtarget->isNeonAvailable())) {
6256 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6257 }
6258 return 0;
6259}
6260
6261unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
6262 switch (VT.SimpleTy) {
6263 case MVT::i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6264 case MVT::i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6265 case MVT::v4i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6266 case MVT::v8i16: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6267 case MVT::v2i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6268 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6269 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6270 default: return 0;
6271 }
6272}
6273
6274// FastEmit functions for ISD::TRUNCATE.
6275
6276unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0) {
6277 if (RetVT.SimpleTy != MVT::i32)
6278 return 0;
6279 return fastEmitInst_extractsubreg(RetVT, Op0, Idx: AArch64::sub_32);
6280}
6281
6282unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6283 if (RetVT.SimpleTy != MVT::v8i8)
6284 return 0;
6285 if ((Subtarget->isNeonAvailable())) {
6286 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv8i8, RC: &AArch64::FPR64RegClass, Op0);
6287 }
6288 return 0;
6289}
6290
6291unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6292 if (RetVT.SimpleTy != MVT::v4i16)
6293 return 0;
6294 if ((Subtarget->isNeonAvailable())) {
6295 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv4i16, RC: &AArch64::FPR64RegClass, Op0);
6296 }
6297 return 0;
6298}
6299
6300unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6301 if (RetVT.SimpleTy != MVT::v2i32)
6302 return 0;
6303 if ((Subtarget->isNeonAvailable())) {
6304 return fastEmitInst_r(MachineInstOpcode: AArch64::XTNv2i32, RC: &AArch64::FPR64RegClass, Op0);
6305 }
6306 return 0;
6307}
6308
6309unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0) {
6310 switch (VT.SimpleTy) {
6311 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0);
6312 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0);
6313 case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0);
6314 case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0);
6315 default: return 0;
6316 }
6317}
6318
6319// FastEmit functions for ISD::UINT_TO_FP.
6320
6321unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0) {
6322 if ((Subtarget->hasFullFP16())) {
6323 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWHri, RC: &AArch64::FPR16RegClass, Op0);
6324 }
6325 return 0;
6326}
6327
6328unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0) {
6329 if ((Subtarget->hasFPARMv8())) {
6330 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWSri, RC: &AArch64::FPR32RegClass, Op0);
6331 }
6332 return 0;
6333}
6334
6335unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0) {
6336 if ((Subtarget->hasFPARMv8())) {
6337 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUWDri, RC: &AArch64::FPR64RegClass, Op0);
6338 }
6339 return 0;
6340}
6341
6342unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0) {
6343switch (RetVT.SimpleTy) {
6344 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0);
6345 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0);
6346 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0);
6347 default: return 0;
6348}
6349}
6350
6351unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0) {
6352 if ((Subtarget->hasFullFP16())) {
6353 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXHri, RC: &AArch64::FPR16RegClass, Op0);
6354 }
6355 return 0;
6356}
6357
6358unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0) {
6359 if ((Subtarget->hasFPARMv8())) {
6360 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXSri, RC: &AArch64::FPR32RegClass, Op0);
6361 }
6362 return 0;
6363}
6364
6365unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0) {
6366 if ((Subtarget->hasFPARMv8())) {
6367 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFUXDri, RC: &AArch64::FPR64RegClass, Op0);
6368 }
6369 return 0;
6370}
6371
6372unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0) {
6373switch (RetVT.SimpleTy) {
6374 case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0);
6375 case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0);
6376 case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0);
6377 default: return 0;
6378}
6379}
6380
6381unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6382 if (RetVT.SimpleTy != MVT::v4f16)
6383 return 0;
6384 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6385 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f16, RC: &AArch64::FPR64RegClass, Op0);
6386 }
6387 return 0;
6388}
6389
6390unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6391 if (RetVT.SimpleTy != MVT::v8f16)
6392 return 0;
6393 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6394 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv8f16, RC: &AArch64::FPR128RegClass, Op0);
6395 }
6396 return 0;
6397}
6398
6399unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
6400 if (RetVT.SimpleTy != MVT::v2f32)
6401 return 0;
6402 if ((Subtarget->isNeonAvailable())) {
6403 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f32, RC: &AArch64::FPR64RegClass, Op0);
6404 }
6405 return 0;
6406}
6407
6408unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6409 if (RetVT.SimpleTy != MVT::v4f32)
6410 return 0;
6411 if ((Subtarget->isNeonAvailable())) {
6412 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv4f32, RC: &AArch64::FPR128RegClass, Op0);
6413 }
6414 return 0;
6415}
6416
6417unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6418 if (RetVT.SimpleTy != MVT::v2f64)
6419 return 0;
6420 if ((Subtarget->isNeonAvailable())) {
6421 return fastEmitInst_r(MachineInstOpcode: AArch64::UCVTFv2f64, RC: &AArch64::FPR128RegClass, Op0);
6422 }
6423 return 0;
6424}
6425
6426unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0) {
6427 switch (VT.SimpleTy) {
6428 case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0);
6429 case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0);
6430 case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0);
6431 case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0);
6432 case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0);
6433 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0);
6434 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0);
6435 default: return 0;
6436 }
6437}
6438
6439// FastEmit functions for ISD::VECREDUCE_ADD.
6440
6441unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
6442 if (RetVT.SimpleTy != MVT::i8)
6443 return 0;
6444 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6445}
6446
6447unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
6448 if (RetVT.SimpleTy != MVT::i8)
6449 return 0;
6450 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6451}
6452
6453unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6454 if (RetVT.SimpleTy != MVT::i16)
6455 return 0;
6456 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6457}
6458
6459unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6460 if (RetVT.SimpleTy != MVT::i16)
6461 return 0;
6462 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6463}
6464
6465unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6466 if (RetVT.SimpleTy != MVT::i32)
6467 return 0;
6468 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6469}
6470
6471unsigned fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
6472 if (RetVT.SimpleTy != MVT::i64)
6473 return 0;
6474 return fastEmitInst_r(MachineInstOpcode: AArch64::ADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6475}
6476
6477unsigned fastEmit_ISD_VECREDUCE_ADD_r(MVT VT, MVT RetVT, unsigned Op0) {
6478 switch (VT.SimpleTy) {
6479 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i8_r(RetVT, Op0);
6480 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_ADD_MVT_v16i8_r(RetVT, Op0);
6481 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i16_r(RetVT, Op0);
6482 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_ADD_MVT_v8i16_r(RetVT, Op0);
6483 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_ADD_MVT_v4i32_r(RetVT, Op0);
6484 case MVT::v2i64: return fastEmit_ISD_VECREDUCE_ADD_MVT_v2i64_r(RetVT, Op0);
6485 default: return 0;
6486 }
6487}
6488
6489// FastEmit functions for ISD::VECREDUCE_FADD.
6490
6491unsigned fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
6492 if (RetVT.SimpleTy != MVT::f32)
6493 return 0;
6494 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6495}
6496
6497unsigned fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6498 if (RetVT.SimpleTy != MVT::f64)
6499 return 0;
6500 return fastEmitInst_r(MachineInstOpcode: AArch64::FADDPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6501}
6502
6503unsigned fastEmit_ISD_VECREDUCE_FADD_r(MVT VT, MVT RetVT, unsigned Op0) {
6504 switch (VT.SimpleTy) {
6505 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f32_r(RetVT, Op0);
6506 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FADD_MVT_v2f64_r(RetVT, Op0);
6507 default: return 0;
6508 }
6509}
6510
6511// FastEmit functions for ISD::VECREDUCE_FMAX.
6512
6513unsigned fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
6514 if (RetVT.SimpleTy != MVT::f16)
6515 return 0;
6516 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6517 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6518 }
6519 return 0;
6520}
6521
6522unsigned fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6523 if (RetVT.SimpleTy != MVT::f16)
6524 return 0;
6525 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6526 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6527 }
6528 return 0;
6529}
6530
6531unsigned fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
6532 if (RetVT.SimpleTy != MVT::f32)
6533 return 0;
6534 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6535}
6536
6537unsigned fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6538 if (RetVT.SimpleTy != MVT::f32)
6539 return 0;
6540 if ((Subtarget->isNeonAvailable())) {
6541 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6542 }
6543 return 0;
6544}
6545
6546unsigned fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6547 if (RetVT.SimpleTy != MVT::f64)
6548 return 0;
6549 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6550}
6551
6552unsigned fastEmit_ISD_VECREDUCE_FMAX_r(MVT VT, MVT RetVT, unsigned Op0) {
6553 switch (VT.SimpleTy) {
6554 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f16_r(RetVT, Op0);
6555 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v8f16_r(RetVT, Op0);
6556 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f32_r(RetVT, Op0);
6557 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v4f32_r(RetVT, Op0);
6558 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAX_MVT_v2f64_r(RetVT, Op0);
6559 default: return 0;
6560 }
6561}
6562
6563// FastEmit functions for ISD::VECREDUCE_FMAXIMUM.
6564
6565unsigned fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
6566 if (RetVT.SimpleTy != MVT::f16)
6567 return 0;
6568 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6569 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6570 }
6571 return 0;
6572}
6573
6574unsigned fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6575 if (RetVT.SimpleTy != MVT::f16)
6576 return 0;
6577 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6578 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6579 }
6580 return 0;
6581}
6582
6583unsigned fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
6584 if (RetVT.SimpleTy != MVT::f32)
6585 return 0;
6586 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6587}
6588
6589unsigned fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6590 if (RetVT.SimpleTy != MVT::f32)
6591 return 0;
6592 if ((Subtarget->isNeonAvailable())) {
6593 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6594 }
6595 return 0;
6596}
6597
6598unsigned fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6599 if (RetVT.SimpleTy != MVT::f64)
6600 return 0;
6601 return fastEmitInst_r(MachineInstOpcode: AArch64::FMAXPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6602}
6603
6604unsigned fastEmit_ISD_VECREDUCE_FMAXIMUM_r(MVT VT, MVT RetVT, unsigned Op0) {
6605 switch (VT.SimpleTy) {
6606 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f16_r(RetVT, Op0);
6607 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v8f16_r(RetVT, Op0);
6608 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f32_r(RetVT, Op0);
6609 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v4f32_r(RetVT, Op0);
6610 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMAXIMUM_MVT_v2f64_r(RetVT, Op0);
6611 default: return 0;
6612 }
6613}
6614
6615// FastEmit functions for ISD::VECREDUCE_FMIN.
6616
6617unsigned fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
6618 if (RetVT.SimpleTy != MVT::f16)
6619 return 0;
6620 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6621 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6622 }
6623 return 0;
6624}
6625
6626unsigned fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6627 if (RetVT.SimpleTy != MVT::f16)
6628 return 0;
6629 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6630 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6631 }
6632 return 0;
6633}
6634
6635unsigned fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
6636 if (RetVT.SimpleTy != MVT::f32)
6637 return 0;
6638 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6639}
6640
6641unsigned fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6642 if (RetVT.SimpleTy != MVT::f32)
6643 return 0;
6644 if ((Subtarget->isNeonAvailable())) {
6645 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6646 }
6647 return 0;
6648}
6649
6650unsigned fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6651 if (RetVT.SimpleTy != MVT::f64)
6652 return 0;
6653 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINNMPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6654}
6655
6656unsigned fastEmit_ISD_VECREDUCE_FMIN_r(MVT VT, MVT RetVT, unsigned Op0) {
6657 switch (VT.SimpleTy) {
6658 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f16_r(RetVT, Op0);
6659 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v8f16_r(RetVT, Op0);
6660 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f32_r(RetVT, Op0);
6661 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v4f32_r(RetVT, Op0);
6662 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMIN_MVT_v2f64_r(RetVT, Op0);
6663 default: return 0;
6664 }
6665}
6666
6667// FastEmit functions for ISD::VECREDUCE_FMINIMUM.
6668
6669unsigned fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
6670 if (RetVT.SimpleTy != MVT::f16)
6671 return 0;
6672 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6673 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6674 }
6675 return 0;
6676}
6677
6678unsigned fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
6679 if (RetVT.SimpleTy != MVT::f16)
6680 return 0;
6681 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
6682 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6683 }
6684 return 0;
6685}
6686
6687unsigned fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
6688 if (RetVT.SimpleTy != MVT::f32)
6689 return 0;
6690 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i32p, RC: &AArch64::FPR32RegClass, Op0);
6691}
6692
6693unsigned fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
6694 if (RetVT.SimpleTy != MVT::f32)
6695 return 0;
6696 if ((Subtarget->isNeonAvailable())) {
6697 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6698 }
6699 return 0;
6700}
6701
6702unsigned fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
6703 if (RetVT.SimpleTy != MVT::f64)
6704 return 0;
6705 return fastEmitInst_r(MachineInstOpcode: AArch64::FMINPv2i64p, RC: &AArch64::FPR64RegClass, Op0);
6706}
6707
6708unsigned fastEmit_ISD_VECREDUCE_FMINIMUM_r(MVT VT, MVT RetVT, unsigned Op0) {
6709 switch (VT.SimpleTy) {
6710 case MVT::v4f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f16_r(RetVT, Op0);
6711 case MVT::v8f16: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v8f16_r(RetVT, Op0);
6712 case MVT::v2f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f32_r(RetVT, Op0);
6713 case MVT::v4f32: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v4f32_r(RetVT, Op0);
6714 case MVT::v2f64: return fastEmit_ISD_VECREDUCE_FMINIMUM_MVT_v2f64_r(RetVT, Op0);
6715 default: return 0;
6716 }
6717}
6718
6719// FastEmit functions for ISD::VECREDUCE_SMAX.
6720
6721unsigned fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
6722 if (RetVT.SimpleTy != MVT::i8)
6723 return 0;
6724 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6725}
6726
6727unsigned fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
6728 if (RetVT.SimpleTy != MVT::i8)
6729 return 0;
6730 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6731}
6732
6733unsigned fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6734 if (RetVT.SimpleTy != MVT::i16)
6735 return 0;
6736 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6737}
6738
6739unsigned fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6740 if (RetVT.SimpleTy != MVT::i16)
6741 return 0;
6742 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6743}
6744
6745unsigned fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6746 if (RetVT.SimpleTy != MVT::i32)
6747 return 0;
6748 return fastEmitInst_r(MachineInstOpcode: AArch64::SMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6749}
6750
6751unsigned fastEmit_ISD_VECREDUCE_SMAX_r(MVT VT, MVT RetVT, unsigned Op0) {
6752 switch (VT.SimpleTy) {
6753 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i8_r(RetVT, Op0);
6754 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v16i8_r(RetVT, Op0);
6755 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i16_r(RetVT, Op0);
6756 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v8i16_r(RetVT, Op0);
6757 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMAX_MVT_v4i32_r(RetVT, Op0);
6758 default: return 0;
6759 }
6760}
6761
6762// FastEmit functions for ISD::VECREDUCE_SMIN.
6763
6764unsigned fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
6765 if (RetVT.SimpleTy != MVT::i8)
6766 return 0;
6767 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6768}
6769
6770unsigned fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
6771 if (RetVT.SimpleTy != MVT::i8)
6772 return 0;
6773 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6774}
6775
6776unsigned fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6777 if (RetVT.SimpleTy != MVT::i16)
6778 return 0;
6779 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6780}
6781
6782unsigned fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6783 if (RetVT.SimpleTy != MVT::i16)
6784 return 0;
6785 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6786}
6787
6788unsigned fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6789 if (RetVT.SimpleTy != MVT::i32)
6790 return 0;
6791 return fastEmitInst_r(MachineInstOpcode: AArch64::SMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6792}
6793
6794unsigned fastEmit_ISD_VECREDUCE_SMIN_r(MVT VT, MVT RetVT, unsigned Op0) {
6795 switch (VT.SimpleTy) {
6796 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i8_r(RetVT, Op0);
6797 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v16i8_r(RetVT, Op0);
6798 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i16_r(RetVT, Op0);
6799 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v8i16_r(RetVT, Op0);
6800 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_SMIN_MVT_v4i32_r(RetVT, Op0);
6801 default: return 0;
6802 }
6803}
6804
6805// FastEmit functions for ISD::VECREDUCE_UMAX.
6806
6807unsigned fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
6808 if (RetVT.SimpleTy != MVT::i8)
6809 return 0;
6810 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6811}
6812
6813unsigned fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
6814 if (RetVT.SimpleTy != MVT::i8)
6815 return 0;
6816 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6817}
6818
6819unsigned fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6820 if (RetVT.SimpleTy != MVT::i16)
6821 return 0;
6822 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6823}
6824
6825unsigned fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6826 if (RetVT.SimpleTy != MVT::i16)
6827 return 0;
6828 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6829}
6830
6831unsigned fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6832 if (RetVT.SimpleTy != MVT::i32)
6833 return 0;
6834 return fastEmitInst_r(MachineInstOpcode: AArch64::UMAXVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6835}
6836
6837unsigned fastEmit_ISD_VECREDUCE_UMAX_r(MVT VT, MVT RetVT, unsigned Op0) {
6838 switch (VT.SimpleTy) {
6839 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i8_r(RetVT, Op0);
6840 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v16i8_r(RetVT, Op0);
6841 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i16_r(RetVT, Op0);
6842 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v8i16_r(RetVT, Op0);
6843 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMAX_MVT_v4i32_r(RetVT, Op0);
6844 default: return 0;
6845 }
6846}
6847
6848// FastEmit functions for ISD::VECREDUCE_UMIN.
6849
6850unsigned fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
6851 if (RetVT.SimpleTy != MVT::i8)
6852 return 0;
6853 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i8v, RC: &AArch64::FPR8RegClass, Op0);
6854}
6855
6856unsigned fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
6857 if (RetVT.SimpleTy != MVT::i8)
6858 return 0;
6859 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv16i8v, RC: &AArch64::FPR8RegClass, Op0);
6860}
6861
6862unsigned fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
6863 if (RetVT.SimpleTy != MVT::i16)
6864 return 0;
6865 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i16v, RC: &AArch64::FPR16RegClass, Op0);
6866}
6867
6868unsigned fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
6869 if (RetVT.SimpleTy != MVT::i16)
6870 return 0;
6871 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv8i16v, RC: &AArch64::FPR16RegClass, Op0);
6872}
6873
6874unsigned fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
6875 if (RetVT.SimpleTy != MVT::i32)
6876 return 0;
6877 return fastEmitInst_r(MachineInstOpcode: AArch64::UMINVv4i32v, RC: &AArch64::FPR32RegClass, Op0);
6878}
6879
6880unsigned fastEmit_ISD_VECREDUCE_UMIN_r(MVT VT, MVT RetVT, unsigned Op0) {
6881 switch (VT.SimpleTy) {
6882 case MVT::v8i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i8_r(RetVT, Op0);
6883 case MVT::v16i8: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v16i8_r(RetVT, Op0);
6884 case MVT::v4i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i16_r(RetVT, Op0);
6885 case MVT::v8i16: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v8i16_r(RetVT, Op0);
6886 case MVT::v4i32: return fastEmit_ISD_VECREDUCE_UMIN_MVT_v4i32_r(RetVT, Op0);
6887 default: return 0;
6888 }
6889}
6890
6891// FastEmit functions for ISD::VECTOR_REVERSE.
6892
6893unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(MVT RetVT, unsigned Op0) {
6894 if (RetVT.SimpleTy != MVT::nxv2i1)
6895 return 0;
6896 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6897 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_D, RC: &AArch64::PPRRegClass, Op0);
6898 }
6899 return 0;
6900}
6901
6902unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(MVT RetVT, unsigned Op0) {
6903 if (RetVT.SimpleTy != MVT::nxv4i1)
6904 return 0;
6905 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6906 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_S, RC: &AArch64::PPRRegClass, Op0);
6907 }
6908 return 0;
6909}
6910
6911unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(MVT RetVT, unsigned Op0) {
6912 if (RetVT.SimpleTy != MVT::nxv8i1)
6913 return 0;
6914 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6915 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_H, RC: &AArch64::PPRRegClass, Op0);
6916 }
6917 return 0;
6918}
6919
6920unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(MVT RetVT, unsigned Op0) {
6921 if (RetVT.SimpleTy != MVT::nxv16i1)
6922 return 0;
6923 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6924 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_PP_B, RC: &AArch64::PPRRegClass, Op0);
6925 }
6926 return 0;
6927}
6928
6929unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(MVT RetVT, unsigned Op0) {
6930 if (RetVT.SimpleTy != MVT::nxv16i8)
6931 return 0;
6932 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6933 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_B, RC: &AArch64::ZPRRegClass, Op0);
6934 }
6935 return 0;
6936}
6937
6938unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(MVT RetVT, unsigned Op0) {
6939 if (RetVT.SimpleTy != MVT::nxv8i16)
6940 return 0;
6941 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6942 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6943 }
6944 return 0;
6945}
6946
6947unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(MVT RetVT, unsigned Op0) {
6948 if (RetVT.SimpleTy != MVT::nxv4i32)
6949 return 0;
6950 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6951 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6952 }
6953 return 0;
6954}
6955
6956unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(MVT RetVT, unsigned Op0) {
6957 if (RetVT.SimpleTy != MVT::nxv2i64)
6958 return 0;
6959 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6960 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6961 }
6962 return 0;
6963}
6964
6965unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(MVT RetVT, unsigned Op0) {
6966 if (RetVT.SimpleTy != MVT::nxv2f16)
6967 return 0;
6968 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6969 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6970 }
6971 return 0;
6972}
6973
6974unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(MVT RetVT, unsigned Op0) {
6975 if (RetVT.SimpleTy != MVT::nxv4f16)
6976 return 0;
6977 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6978 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
6979 }
6980 return 0;
6981}
6982
6983unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(MVT RetVT, unsigned Op0) {
6984 if (RetVT.SimpleTy != MVT::nxv8f16)
6985 return 0;
6986 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6987 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
6988 }
6989 return 0;
6990}
6991
6992unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(MVT RetVT, unsigned Op0) {
6993 if (RetVT.SimpleTy != MVT::nxv2bf16)
6994 return 0;
6995 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
6996 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
6997 }
6998 return 0;
6999}
7000
7001unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(MVT RetVT, unsigned Op0) {
7002 if (RetVT.SimpleTy != MVT::nxv4bf16)
7003 return 0;
7004 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7005 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
7006 }
7007 return 0;
7008}
7009
7010unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(MVT RetVT, unsigned Op0) {
7011 if (RetVT.SimpleTy != MVT::nxv8bf16)
7012 return 0;
7013 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7014 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_H, RC: &AArch64::ZPRRegClass, Op0);
7015 }
7016 return 0;
7017}
7018
7019unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(MVT RetVT, unsigned Op0) {
7020 if (RetVT.SimpleTy != MVT::nxv2f32)
7021 return 0;
7022 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7023 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
7024 }
7025 return 0;
7026}
7027
7028unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(MVT RetVT, unsigned Op0) {
7029 if (RetVT.SimpleTy != MVT::nxv4f32)
7030 return 0;
7031 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7032 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_S, RC: &AArch64::ZPRRegClass, Op0);
7033 }
7034 return 0;
7035}
7036
7037unsigned fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(MVT RetVT, unsigned Op0) {
7038 if (RetVT.SimpleTy != MVT::nxv2f64)
7039 return 0;
7040 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7041 return fastEmitInst_r(MachineInstOpcode: AArch64::REV_ZZ_D, RC: &AArch64::ZPRRegClass, Op0);
7042 }
7043 return 0;
7044}
7045
7046unsigned fastEmit_ISD_VECTOR_REVERSE_r(MVT VT, MVT RetVT, unsigned Op0) {
7047 switch (VT.SimpleTy) {
7048 case MVT::nxv2i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i1_r(RetVT, Op0);
7049 case MVT::nxv4i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i1_r(RetVT, Op0);
7050 case MVT::nxv8i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i1_r(RetVT, Op0);
7051 case MVT::nxv16i1: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i1_r(RetVT, Op0);
7052 case MVT::nxv16i8: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv16i8_r(RetVT, Op0);
7053 case MVT::nxv8i16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8i16_r(RetVT, Op0);
7054 case MVT::nxv4i32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4i32_r(RetVT, Op0);
7055 case MVT::nxv2i64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2i64_r(RetVT, Op0);
7056 case MVT::nxv2f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f16_r(RetVT, Op0);
7057 case MVT::nxv4f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f16_r(RetVT, Op0);
7058 case MVT::nxv8f16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8f16_r(RetVT, Op0);
7059 case MVT::nxv2bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2bf16_r(RetVT, Op0);
7060 case MVT::nxv4bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4bf16_r(RetVT, Op0);
7061 case MVT::nxv8bf16: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv8bf16_r(RetVT, Op0);
7062 case MVT::nxv2f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f32_r(RetVT, Op0);
7063 case MVT::nxv4f32: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv4f32_r(RetVT, Op0);
7064 case MVT::nxv2f64: return fastEmit_ISD_VECTOR_REVERSE_MVT_nxv2f64_r(RetVT, Op0);
7065 default: return 0;
7066 }
7067}
7068
7069// Top-level FastEmit function.
7070
7071unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0) override {
7072 switch (Opcode) {
7073 case AArch64ISD::ALLOCATE_ZA_BUFFER: return fastEmit_AArch64ISD_ALLOCATE_ZA_BUFFER_r(VT, RetVT, Op0);
7074 case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0);
7075 case AArch64ISD::CMEQz: return fastEmit_AArch64ISD_CMEQz_r(VT, RetVT, Op0);
7076 case AArch64ISD::CMGEz: return fastEmit_AArch64ISD_CMGEz_r(VT, RetVT, Op0);
7077 case AArch64ISD::CMGTz: return fastEmit_AArch64ISD_CMGTz_r(VT, RetVT, Op0);
7078 case AArch64ISD::CMLEz: return fastEmit_AArch64ISD_CMLEz_r(VT, RetVT, Op0);
7079 case AArch64ISD::CMLTz: return fastEmit_AArch64ISD_CMLTz_r(VT, RetVT, Op0);
7080 case AArch64ISD::COALESCER_BARRIER: return fastEmit_AArch64ISD_COALESCER_BARRIER_r(VT, RetVT, Op0);
7081 case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0);
7082 case AArch64ISD::FCMEQz: return fastEmit_AArch64ISD_FCMEQz_r(VT, RetVT, Op0);
7083 case AArch64ISD::FCMGEz: return fastEmit_AArch64ISD_FCMGEz_r(VT, RetVT, Op0);
7084 case AArch64ISD::FCMGTz: return fastEmit_AArch64ISD_FCMGTz_r(VT, RetVT, Op0);
7085 case AArch64ISD::FCMLEz: return fastEmit_AArch64ISD_FCMLEz_r(VT, RetVT, Op0);
7086 case AArch64ISD::FCMLTz: return fastEmit_AArch64ISD_FCMLTz_r(VT, RetVT, Op0);
7087 case AArch64ISD::FCVTXN: return fastEmit_AArch64ISD_FCVTXN_r(VT, RetVT, Op0);
7088 case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0);
7089 case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0);
7090 case AArch64ISD::INIT_TPIDR2OBJ: return fastEmit_AArch64ISD_INIT_TPIDR2OBJ_r(VT, RetVT, Op0);
7091 case AArch64ISD::PROBED_ALLOCA: return fastEmit_AArch64ISD_PROBED_ALLOCA_r(VT, RetVT, Op0);
7092 case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0);
7093 case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0);
7094 case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0);
7095 case AArch64ISD::SADDLP: return fastEmit_AArch64ISD_SADDLP_r(VT, RetVT, Op0);
7096 case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0);
7097 case AArch64ISD::SUNPKHI: return fastEmit_AArch64ISD_SUNPKHI_r(VT, RetVT, Op0);
7098 case AArch64ISD::SUNPKLO: return fastEmit_AArch64ISD_SUNPKLO_r(VT, RetVT, Op0);
7099 case AArch64ISD::UADDLP: return fastEmit_AArch64ISD_UADDLP_r(VT, RetVT, Op0);
7100 case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0);
7101 case AArch64ISD::UUNPKHI: return fastEmit_AArch64ISD_UUNPKHI_r(VT, RetVT, Op0);
7102 case AArch64ISD::UUNPKLO: return fastEmit_AArch64ISD_UUNPKLO_r(VT, RetVT, Op0);
7103 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0);
7104 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0);
7105 case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0);
7106 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0);
7107 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0);
7108 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0);
7109 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0);
7110 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0);
7111 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0);
7112 case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0);
7113 case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0);
7114 case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0);
7115 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0);
7116 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0);
7117 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0);
7118 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0);
7119 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0);
7120 case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0);
7121 case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0);
7122 case ISD::FROUNDEVEN: return fastEmit_ISD_FROUNDEVEN_r(VT, RetVT, Op0);
7123 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0);
7124 case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0);
7125 case ISD::LLROUND: return fastEmit_ISD_LLROUND_r(VT, RetVT, Op0);
7126 case ISD::LROUND: return fastEmit_ISD_LROUND_r(VT, RetVT, Op0);
7127 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0);
7128 case ISD::SPLAT_VECTOR: return fastEmit_ISD_SPLAT_VECTOR_r(VT, RetVT, Op0);
7129 case ISD::STRICT_FCEIL: return fastEmit_ISD_STRICT_FCEIL_r(VT, RetVT, Op0);
7130 case ISD::STRICT_FFLOOR: return fastEmit_ISD_STRICT_FFLOOR_r(VT, RetVT, Op0);
7131 case ISD::STRICT_FNEARBYINT: return fastEmit_ISD_STRICT_FNEARBYINT_r(VT, RetVT, Op0);
7132 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0);
7133 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0);
7134 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0);
7135 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0);
7136 case ISD::STRICT_FRINT: return fastEmit_ISD_STRICT_FRINT_r(VT, RetVT, Op0);
7137 case ISD::STRICT_FROUND: return fastEmit_ISD_STRICT_FROUND_r(VT, RetVT, Op0);
7138 case ISD::STRICT_FROUNDEVEN: return fastEmit_ISD_STRICT_FROUNDEVEN_r(VT, RetVT, Op0);
7139 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0);
7140 case ISD::STRICT_FTRUNC: return fastEmit_ISD_STRICT_FTRUNC_r(VT, RetVT, Op0);
7141 case ISD::STRICT_LLROUND: return fastEmit_ISD_STRICT_LLROUND_r(VT, RetVT, Op0);
7142 case ISD::STRICT_LROUND: return fastEmit_ISD_STRICT_LROUND_r(VT, RetVT, Op0);
7143 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0);
7144 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0);
7145 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0);
7146 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0);
7147 case ISD::VECREDUCE_ADD: return fastEmit_ISD_VECREDUCE_ADD_r(VT, RetVT, Op0);
7148 case ISD::VECREDUCE_FADD: return fastEmit_ISD_VECREDUCE_FADD_r(VT, RetVT, Op0);
7149 case ISD::VECREDUCE_FMAX: return fastEmit_ISD_VECREDUCE_FMAX_r(VT, RetVT, Op0);
7150 case ISD::VECREDUCE_FMAXIMUM: return fastEmit_ISD_VECREDUCE_FMAXIMUM_r(VT, RetVT, Op0);
7151 case ISD::VECREDUCE_FMIN: return fastEmit_ISD_VECREDUCE_FMIN_r(VT, RetVT, Op0);
7152 case ISD::VECREDUCE_FMINIMUM: return fastEmit_ISD_VECREDUCE_FMINIMUM_r(VT, RetVT, Op0);
7153 case ISD::VECREDUCE_SMAX: return fastEmit_ISD_VECREDUCE_SMAX_r(VT, RetVT, Op0);
7154 case ISD::VECREDUCE_SMIN: return fastEmit_ISD_VECREDUCE_SMIN_r(VT, RetVT, Op0);
7155 case ISD::VECREDUCE_UMAX: return fastEmit_ISD_VECREDUCE_UMAX_r(VT, RetVT, Op0);
7156 case ISD::VECREDUCE_UMIN: return fastEmit_ISD_VECREDUCE_UMIN_r(VT, RetVT, Op0);
7157 case ISD::VECTOR_REVERSE: return fastEmit_ISD_VECTOR_REVERSE_r(VT, RetVT, Op0);
7158 default: return 0;
7159 }
7160}
7161
7162// FastEmit functions for AArch64ISD::ADDP.
7163
7164unsigned fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7165 if (RetVT.SimpleTy != MVT::v8i8)
7166 return 0;
7167 if ((Subtarget->isNeonAvailable())) {
7168 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7169 }
7170 return 0;
7171}
7172
7173unsigned fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7174 if (RetVT.SimpleTy != MVT::v16i8)
7175 return 0;
7176 if ((Subtarget->isNeonAvailable())) {
7177 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7178 }
7179 return 0;
7180}
7181
7182unsigned fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7183 if (RetVT.SimpleTy != MVT::v4i16)
7184 return 0;
7185 if ((Subtarget->isNeonAvailable())) {
7186 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7187 }
7188 return 0;
7189}
7190
7191unsigned fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7192 if (RetVT.SimpleTy != MVT::v8i16)
7193 return 0;
7194 if ((Subtarget->isNeonAvailable())) {
7195 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7196 }
7197 return 0;
7198}
7199
7200unsigned fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7201 if (RetVT.SimpleTy != MVT::v2i32)
7202 return 0;
7203 if ((Subtarget->isNeonAvailable())) {
7204 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7205 }
7206 return 0;
7207}
7208
7209unsigned fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7210 if (RetVT.SimpleTy != MVT::v4i32)
7211 return 0;
7212 if ((Subtarget->isNeonAvailable())) {
7213 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7214 }
7215 return 0;
7216}
7217
7218unsigned fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7219 if (RetVT.SimpleTy != MVT::v2i64)
7220 return 0;
7221 if ((Subtarget->isNeonAvailable())) {
7222 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDPv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7223 }
7224 return 0;
7225}
7226
7227unsigned fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7228 if (RetVT.SimpleTy != MVT::v4f16)
7229 return 0;
7230 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7231 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7232 }
7233 return 0;
7234}
7235
7236unsigned fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7237 if (RetVT.SimpleTy != MVT::v8f16)
7238 return 0;
7239 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7240 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7241 }
7242 return 0;
7243}
7244
7245unsigned fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7246 if (RetVT.SimpleTy != MVT::v2f32)
7247 return 0;
7248 if ((Subtarget->isNeonAvailable())) {
7249 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7250 }
7251 return 0;
7252}
7253
7254unsigned fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7255 if (RetVT.SimpleTy != MVT::v4f32)
7256 return 0;
7257 if ((Subtarget->isNeonAvailable())) {
7258 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7259 }
7260 return 0;
7261}
7262
7263unsigned fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7264 if (RetVT.SimpleTy != MVT::v2f64)
7265 return 0;
7266 if ((Subtarget->isNeonAvailable())) {
7267 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDPv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7268 }
7269 return 0;
7270}
7271
7272unsigned fastEmit_AArch64ISD_ADDP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7273 switch (VT.SimpleTy) {
7274 case MVT::v8i8: return fastEmit_AArch64ISD_ADDP_MVT_v8i8_rr(RetVT, Op0, Op1);
7275 case MVT::v16i8: return fastEmit_AArch64ISD_ADDP_MVT_v16i8_rr(RetVT, Op0, Op1);
7276 case MVT::v4i16: return fastEmit_AArch64ISD_ADDP_MVT_v4i16_rr(RetVT, Op0, Op1);
7277 case MVT::v8i16: return fastEmit_AArch64ISD_ADDP_MVT_v8i16_rr(RetVT, Op0, Op1);
7278 case MVT::v2i32: return fastEmit_AArch64ISD_ADDP_MVT_v2i32_rr(RetVT, Op0, Op1);
7279 case MVT::v4i32: return fastEmit_AArch64ISD_ADDP_MVT_v4i32_rr(RetVT, Op0, Op1);
7280 case MVT::v2i64: return fastEmit_AArch64ISD_ADDP_MVT_v2i64_rr(RetVT, Op0, Op1);
7281 case MVT::v4f16: return fastEmit_AArch64ISD_ADDP_MVT_v4f16_rr(RetVT, Op0, Op1);
7282 case MVT::v8f16: return fastEmit_AArch64ISD_ADDP_MVT_v8f16_rr(RetVT, Op0, Op1);
7283 case MVT::v2f32: return fastEmit_AArch64ISD_ADDP_MVT_v2f32_rr(RetVT, Op0, Op1);
7284 case MVT::v4f32: return fastEmit_AArch64ISD_ADDP_MVT_v4f32_rr(RetVT, Op0, Op1);
7285 case MVT::v2f64: return fastEmit_AArch64ISD_ADDP_MVT_v2f64_rr(RetVT, Op0, Op1);
7286 default: return 0;
7287 }
7288}
7289
7290// FastEmit functions for AArch64ISD::BIC.
7291
7292unsigned fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7293 if (RetVT.SimpleTy != MVT::nxv16i8)
7294 return 0;
7295 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7296 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7297 }
7298 return 0;
7299}
7300
7301unsigned fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7302 if (RetVT.SimpleTy != MVT::nxv8i16)
7303 return 0;
7304 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7305 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7306 }
7307 return 0;
7308}
7309
7310unsigned fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7311 if (RetVT.SimpleTy != MVT::nxv4i32)
7312 return 0;
7313 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7314 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7315 }
7316 return 0;
7317}
7318
7319unsigned fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7320 if (RetVT.SimpleTy != MVT::nxv2i64)
7321 return 0;
7322 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
7323 return fastEmitInst_rr(MachineInstOpcode: AArch64::BIC_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
7324 }
7325 return 0;
7326}
7327
7328unsigned fastEmit_AArch64ISD_BIC_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7329 switch (VT.SimpleTy) {
7330 case MVT::nxv16i8: return fastEmit_AArch64ISD_BIC_MVT_nxv16i8_rr(RetVT, Op0, Op1);
7331 case MVT::nxv8i16: return fastEmit_AArch64ISD_BIC_MVT_nxv8i16_rr(RetVT, Op0, Op1);
7332 case MVT::nxv4i32: return fastEmit_AArch64ISD_BIC_MVT_nxv4i32_rr(RetVT, Op0, Op1);
7333 case MVT::nxv2i64: return fastEmit_AArch64ISD_BIC_MVT_nxv2i64_rr(RetVT, Op0, Op1);
7334 default: return 0;
7335 }
7336}
7337
7338// FastEmit functions for AArch64ISD::CMEQ.
7339
7340unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7341 if (RetVT.SimpleTy != MVT::v8i8)
7342 return 0;
7343 if ((Subtarget->isNeonAvailable())) {
7344 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7345 }
7346 return 0;
7347}
7348
7349unsigned fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7350 if (RetVT.SimpleTy != MVT::v16i8)
7351 return 0;
7352 if ((Subtarget->isNeonAvailable())) {
7353 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7354 }
7355 return 0;
7356}
7357
7358unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7359 if (RetVT.SimpleTy != MVT::v4i16)
7360 return 0;
7361 if ((Subtarget->isNeonAvailable())) {
7362 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7363 }
7364 return 0;
7365}
7366
7367unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7368 if (RetVT.SimpleTy != MVT::v8i16)
7369 return 0;
7370 if ((Subtarget->isNeonAvailable())) {
7371 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7372 }
7373 return 0;
7374}
7375
7376unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7377 if (RetVT.SimpleTy != MVT::v2i32)
7378 return 0;
7379 if ((Subtarget->isNeonAvailable())) {
7380 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7381 }
7382 return 0;
7383}
7384
7385unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7386 if (RetVT.SimpleTy != MVT::v4i32)
7387 return 0;
7388 if ((Subtarget->isNeonAvailable())) {
7389 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7390 }
7391 return 0;
7392}
7393
7394unsigned fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7395 if (RetVT.SimpleTy != MVT::v1i64)
7396 return 0;
7397 if ((Subtarget->isNeonAvailable())) {
7398 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7399 }
7400 return 0;
7401}
7402
7403unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7404 if (RetVT.SimpleTy != MVT::v2i64)
7405 return 0;
7406 if ((Subtarget->isNeonAvailable())) {
7407 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMEQv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7408 }
7409 return 0;
7410}
7411
7412unsigned fastEmit_AArch64ISD_CMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7413 switch (VT.SimpleTy) {
7414 case MVT::v8i8: return fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(RetVT, Op0, Op1);
7415 case MVT::v16i8: return fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(RetVT, Op0, Op1);
7416 case MVT::v4i16: return fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(RetVT, Op0, Op1);
7417 case MVT::v8i16: return fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(RetVT, Op0, Op1);
7418 case MVT::v2i32: return fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(RetVT, Op0, Op1);
7419 case MVT::v4i32: return fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(RetVT, Op0, Op1);
7420 case MVT::v1i64: return fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(RetVT, Op0, Op1);
7421 case MVT::v2i64: return fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(RetVT, Op0, Op1);
7422 default: return 0;
7423 }
7424}
7425
7426// FastEmit functions for AArch64ISD::CMGE.
7427
7428unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7429 if (RetVT.SimpleTy != MVT::v8i8)
7430 return 0;
7431 if ((Subtarget->isNeonAvailable())) {
7432 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7433 }
7434 return 0;
7435}
7436
7437unsigned fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7438 if (RetVT.SimpleTy != MVT::v16i8)
7439 return 0;
7440 if ((Subtarget->isNeonAvailable())) {
7441 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7442 }
7443 return 0;
7444}
7445
7446unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7447 if (RetVT.SimpleTy != MVT::v4i16)
7448 return 0;
7449 if ((Subtarget->isNeonAvailable())) {
7450 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7451 }
7452 return 0;
7453}
7454
7455unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7456 if (RetVT.SimpleTy != MVT::v8i16)
7457 return 0;
7458 if ((Subtarget->isNeonAvailable())) {
7459 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7460 }
7461 return 0;
7462}
7463
7464unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7465 if (RetVT.SimpleTy != MVT::v2i32)
7466 return 0;
7467 if ((Subtarget->isNeonAvailable())) {
7468 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7469 }
7470 return 0;
7471}
7472
7473unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7474 if (RetVT.SimpleTy != MVT::v4i32)
7475 return 0;
7476 if ((Subtarget->isNeonAvailable())) {
7477 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7478 }
7479 return 0;
7480}
7481
7482unsigned fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7483 if (RetVT.SimpleTy != MVT::v1i64)
7484 return 0;
7485 if ((Subtarget->isNeonAvailable())) {
7486 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7487 }
7488 return 0;
7489}
7490
7491unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7492 if (RetVT.SimpleTy != MVT::v2i64)
7493 return 0;
7494 if ((Subtarget->isNeonAvailable())) {
7495 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGEv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7496 }
7497 return 0;
7498}
7499
7500unsigned fastEmit_AArch64ISD_CMGE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7501 switch (VT.SimpleTy) {
7502 case MVT::v8i8: return fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(RetVT, Op0, Op1);
7503 case MVT::v16i8: return fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(RetVT, Op0, Op1);
7504 case MVT::v4i16: return fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(RetVT, Op0, Op1);
7505 case MVT::v8i16: return fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(RetVT, Op0, Op1);
7506 case MVT::v2i32: return fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(RetVT, Op0, Op1);
7507 case MVT::v4i32: return fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(RetVT, Op0, Op1);
7508 case MVT::v1i64: return fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(RetVT, Op0, Op1);
7509 case MVT::v2i64: return fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(RetVT, Op0, Op1);
7510 default: return 0;
7511 }
7512}
7513
7514// FastEmit functions for AArch64ISD::CMGT.
7515
7516unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7517 if (RetVT.SimpleTy != MVT::v8i8)
7518 return 0;
7519 if ((Subtarget->isNeonAvailable())) {
7520 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7521 }
7522 return 0;
7523}
7524
7525unsigned fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7526 if (RetVT.SimpleTy != MVT::v16i8)
7527 return 0;
7528 if ((Subtarget->isNeonAvailable())) {
7529 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7530 }
7531 return 0;
7532}
7533
7534unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7535 if (RetVT.SimpleTy != MVT::v4i16)
7536 return 0;
7537 if ((Subtarget->isNeonAvailable())) {
7538 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7539 }
7540 return 0;
7541}
7542
7543unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7544 if (RetVT.SimpleTy != MVT::v8i16)
7545 return 0;
7546 if ((Subtarget->isNeonAvailable())) {
7547 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7548 }
7549 return 0;
7550}
7551
7552unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7553 if (RetVT.SimpleTy != MVT::v2i32)
7554 return 0;
7555 if ((Subtarget->isNeonAvailable())) {
7556 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7557 }
7558 return 0;
7559}
7560
7561unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7562 if (RetVT.SimpleTy != MVT::v4i32)
7563 return 0;
7564 if ((Subtarget->isNeonAvailable())) {
7565 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7566 }
7567 return 0;
7568}
7569
7570unsigned fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7571 if (RetVT.SimpleTy != MVT::v1i64)
7572 return 0;
7573 if ((Subtarget->isNeonAvailable())) {
7574 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7575 }
7576 return 0;
7577}
7578
7579unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7580 if (RetVT.SimpleTy != MVT::v2i64)
7581 return 0;
7582 if ((Subtarget->isNeonAvailable())) {
7583 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMGTv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7584 }
7585 return 0;
7586}
7587
7588unsigned fastEmit_AArch64ISD_CMGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7589 switch (VT.SimpleTy) {
7590 case MVT::v8i8: return fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(RetVT, Op0, Op1);
7591 case MVT::v16i8: return fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(RetVT, Op0, Op1);
7592 case MVT::v4i16: return fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(RetVT, Op0, Op1);
7593 case MVT::v8i16: return fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(RetVT, Op0, Op1);
7594 case MVT::v2i32: return fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(RetVT, Op0, Op1);
7595 case MVT::v4i32: return fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(RetVT, Op0, Op1);
7596 case MVT::v1i64: return fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(RetVT, Op0, Op1);
7597 case MVT::v2i64: return fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(RetVT, Op0, Op1);
7598 default: return 0;
7599 }
7600}
7601
7602// FastEmit functions for AArch64ISD::CMHI.
7603
7604unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7605 if (RetVT.SimpleTy != MVT::v8i8)
7606 return 0;
7607 if ((Subtarget->isNeonAvailable())) {
7608 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7609 }
7610 return 0;
7611}
7612
7613unsigned fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7614 if (RetVT.SimpleTy != MVT::v16i8)
7615 return 0;
7616 if ((Subtarget->isNeonAvailable())) {
7617 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7618 }
7619 return 0;
7620}
7621
7622unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7623 if (RetVT.SimpleTy != MVT::v4i16)
7624 return 0;
7625 if ((Subtarget->isNeonAvailable())) {
7626 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7627 }
7628 return 0;
7629}
7630
7631unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7632 if (RetVT.SimpleTy != MVT::v8i16)
7633 return 0;
7634 if ((Subtarget->isNeonAvailable())) {
7635 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7636 }
7637 return 0;
7638}
7639
7640unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7641 if (RetVT.SimpleTy != MVT::v2i32)
7642 return 0;
7643 if ((Subtarget->isNeonAvailable())) {
7644 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7645 }
7646 return 0;
7647}
7648
7649unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7650 if (RetVT.SimpleTy != MVT::v4i32)
7651 return 0;
7652 if ((Subtarget->isNeonAvailable())) {
7653 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7654 }
7655 return 0;
7656}
7657
7658unsigned fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7659 if (RetVT.SimpleTy != MVT::v1i64)
7660 return 0;
7661 if ((Subtarget->isNeonAvailable())) {
7662 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7663 }
7664 return 0;
7665}
7666
7667unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7668 if (RetVT.SimpleTy != MVT::v2i64)
7669 return 0;
7670 if ((Subtarget->isNeonAvailable())) {
7671 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHIv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7672 }
7673 return 0;
7674}
7675
7676unsigned fastEmit_AArch64ISD_CMHI_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7677 switch (VT.SimpleTy) {
7678 case MVT::v8i8: return fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(RetVT, Op0, Op1);
7679 case MVT::v16i8: return fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(RetVT, Op0, Op1);
7680 case MVT::v4i16: return fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(RetVT, Op0, Op1);
7681 case MVT::v8i16: return fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(RetVT, Op0, Op1);
7682 case MVT::v2i32: return fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(RetVT, Op0, Op1);
7683 case MVT::v4i32: return fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(RetVT, Op0, Op1);
7684 case MVT::v1i64: return fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(RetVT, Op0, Op1);
7685 case MVT::v2i64: return fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(RetVT, Op0, Op1);
7686 default: return 0;
7687 }
7688}
7689
7690// FastEmit functions for AArch64ISD::CMHS.
7691
7692unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7693 if (RetVT.SimpleTy != MVT::v8i8)
7694 return 0;
7695 if ((Subtarget->isNeonAvailable())) {
7696 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
7697 }
7698 return 0;
7699}
7700
7701unsigned fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7702 if (RetVT.SimpleTy != MVT::v16i8)
7703 return 0;
7704 if ((Subtarget->isNeonAvailable())) {
7705 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
7706 }
7707 return 0;
7708}
7709
7710unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7711 if (RetVT.SimpleTy != MVT::v4i16)
7712 return 0;
7713 if ((Subtarget->isNeonAvailable())) {
7714 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7715 }
7716 return 0;
7717}
7718
7719unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7720 if (RetVT.SimpleTy != MVT::v8i16)
7721 return 0;
7722 if ((Subtarget->isNeonAvailable())) {
7723 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7724 }
7725 return 0;
7726}
7727
7728unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7729 if (RetVT.SimpleTy != MVT::v2i32)
7730 return 0;
7731 if ((Subtarget->isNeonAvailable())) {
7732 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7733 }
7734 return 0;
7735}
7736
7737unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7738 if (RetVT.SimpleTy != MVT::v4i32)
7739 return 0;
7740 if ((Subtarget->isNeonAvailable())) {
7741 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7742 }
7743 return 0;
7744}
7745
7746unsigned fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7747 if (RetVT.SimpleTy != MVT::v1i64)
7748 return 0;
7749 if ((Subtarget->isNeonAvailable())) {
7750 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7751 }
7752 return 0;
7753}
7754
7755unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7756 if (RetVT.SimpleTy != MVT::v2i64)
7757 return 0;
7758 if ((Subtarget->isNeonAvailable())) {
7759 return fastEmitInst_rr(MachineInstOpcode: AArch64::CMHSv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7760 }
7761 return 0;
7762}
7763
7764unsigned fastEmit_AArch64ISD_CMHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7765 switch (VT.SimpleTy) {
7766 case MVT::v8i8: return fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(RetVT, Op0, Op1);
7767 case MVT::v16i8: return fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(RetVT, Op0, Op1);
7768 case MVT::v4i16: return fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(RetVT, Op0, Op1);
7769 case MVT::v8i16: return fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(RetVT, Op0, Op1);
7770 case MVT::v2i32: return fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(RetVT, Op0, Op1);
7771 case MVT::v4i32: return fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(RetVT, Op0, Op1);
7772 case MVT::v1i64: return fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(RetVT, Op0, Op1);
7773 case MVT::v2i64: return fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(RetVT, Op0, Op1);
7774 default: return 0;
7775 }
7776}
7777
7778// FastEmit functions for AArch64ISD::FCMEQ.
7779
7780unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7781 if (RetVT.SimpleTy != MVT::i32)
7782 return 0;
7783 if ((Subtarget->isNeonAvailable())) {
7784 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7785 }
7786 return 0;
7787}
7788
7789unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7790 if (RetVT.SimpleTy != MVT::i64)
7791 return 0;
7792 if ((Subtarget->isNeonAvailable())) {
7793 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7794 }
7795 return 0;
7796}
7797
7798unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7799 if (RetVT.SimpleTy != MVT::v4i16)
7800 return 0;
7801 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7802 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7803 }
7804 return 0;
7805}
7806
7807unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7808 if (RetVT.SimpleTy != MVT::v8i16)
7809 return 0;
7810 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7811 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7812 }
7813 return 0;
7814}
7815
7816unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7817 if (RetVT.SimpleTy != MVT::v2i32)
7818 return 0;
7819 if ((Subtarget->isNeonAvailable())) {
7820 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7821 }
7822 return 0;
7823}
7824
7825unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7826 if (RetVT.SimpleTy != MVT::v4i32)
7827 return 0;
7828 if ((Subtarget->isNeonAvailable())) {
7829 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7830 }
7831 return 0;
7832}
7833
7834unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7835 if (RetVT.SimpleTy != MVT::v1i64)
7836 return 0;
7837 if ((Subtarget->isNeonAvailable())) {
7838 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQ64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7839 }
7840 return 0;
7841}
7842
7843unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7844 if (RetVT.SimpleTy != MVT::v2i64)
7845 return 0;
7846 if ((Subtarget->isNeonAvailable())) {
7847 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMEQv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7848 }
7849 return 0;
7850}
7851
7852unsigned fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7853 switch (VT.SimpleTy) {
7854 case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op1);
7855 case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op1);
7856 case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op1);
7857 case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op1);
7858 case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op1);
7859 case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op1);
7860 case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op1);
7861 case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op1);
7862 default: return 0;
7863 }
7864}
7865
7866// FastEmit functions for AArch64ISD::FCMGE.
7867
7868unsigned fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7869 if (RetVT.SimpleTy != MVT::i32)
7870 return 0;
7871 if ((Subtarget->isNeonAvailable())) {
7872 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7873 }
7874 return 0;
7875}
7876
7877unsigned fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7878 if (RetVT.SimpleTy != MVT::i64)
7879 return 0;
7880 if ((Subtarget->isNeonAvailable())) {
7881 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7882 }
7883 return 0;
7884}
7885
7886unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7887 if (RetVT.SimpleTy != MVT::v4i16)
7888 return 0;
7889 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7890 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7891 }
7892 return 0;
7893}
7894
7895unsigned fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7896 if (RetVT.SimpleTy != MVT::v8i16)
7897 return 0;
7898 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7899 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7900 }
7901 return 0;
7902}
7903
7904unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7905 if (RetVT.SimpleTy != MVT::v2i32)
7906 return 0;
7907 if ((Subtarget->isNeonAvailable())) {
7908 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7909 }
7910 return 0;
7911}
7912
7913unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7914 if (RetVT.SimpleTy != MVT::v4i32)
7915 return 0;
7916 if ((Subtarget->isNeonAvailable())) {
7917 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
7918 }
7919 return 0;
7920}
7921
7922unsigned fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7923 if (RetVT.SimpleTy != MVT::v1i64)
7924 return 0;
7925 if ((Subtarget->isNeonAvailable())) {
7926 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGE64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7927 }
7928 return 0;
7929}
7930
7931unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7932 if (RetVT.SimpleTy != MVT::v2i64)
7933 return 0;
7934 if ((Subtarget->isNeonAvailable())) {
7935 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGEv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
7936 }
7937 return 0;
7938}
7939
7940unsigned fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
7941 switch (VT.SimpleTy) {
7942 case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op1);
7943 case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op1);
7944 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op1);
7945 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op1);
7946 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op1);
7947 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op1);
7948 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op1);
7949 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op1);
7950 default: return 0;
7951 }
7952}
7953
7954// FastEmit functions for AArch64ISD::FCMGT.
7955
7956unsigned fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7957 if (RetVT.SimpleTy != MVT::i32)
7958 return 0;
7959 if ((Subtarget->isNeonAvailable())) {
7960 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT32, RC: &AArch64::FPR32RegClass, Op0, Op1);
7961 }
7962 return 0;
7963}
7964
7965unsigned fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7966 if (RetVT.SimpleTy != MVT::i64)
7967 return 0;
7968 if ((Subtarget->isNeonAvailable())) {
7969 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
7970 }
7971 return 0;
7972}
7973
7974unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7975 if (RetVT.SimpleTy != MVT::v4i16)
7976 return 0;
7977 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7978 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
7979 }
7980 return 0;
7981}
7982
7983unsigned fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7984 if (RetVT.SimpleTy != MVT::v8i16)
7985 return 0;
7986 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
7987 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
7988 }
7989 return 0;
7990}
7991
7992unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
7993 if (RetVT.SimpleTy != MVT::v2i32)
7994 return 0;
7995 if ((Subtarget->isNeonAvailable())) {
7996 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
7997 }
7998 return 0;
7999}
8000
8001unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8002 if (RetVT.SimpleTy != MVT::v4i32)
8003 return 0;
8004 if ((Subtarget->isNeonAvailable())) {
8005 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8006 }
8007 return 0;
8008}
8009
8010unsigned fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8011 if (RetVT.SimpleTy != MVT::v1i64)
8012 return 0;
8013 if ((Subtarget->isNeonAvailable())) {
8014 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGT64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8015 }
8016 return 0;
8017}
8018
8019unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8020 if (RetVT.SimpleTy != MVT::v2i64)
8021 return 0;
8022 if ((Subtarget->isNeonAvailable())) {
8023 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMGTv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8024 }
8025 return 0;
8026}
8027
8028unsigned fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8029 switch (VT.SimpleTy) {
8030 case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op1);
8031 case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op1);
8032 case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op1);
8033 case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op1);
8034 case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op1);
8035 case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op1);
8036 case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op1);
8037 case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op1);
8038 default: return 0;
8039 }
8040}
8041
8042// FastEmit functions for AArch64ISD::FCMP.
8043
8044unsigned fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8045 if (RetVT.SimpleTy != MVT::isVoid)
8046 return 0;
8047 if ((Subtarget->hasFullFP16())) {
8048 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
8049 }
8050 return 0;
8051}
8052
8053unsigned fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8054 if (RetVT.SimpleTy != MVT::isVoid)
8055 return 0;
8056 if ((Subtarget->hasFPARMv8())) {
8057 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
8058 }
8059 return 0;
8060}
8061
8062unsigned fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8063 if (RetVT.SimpleTy != MVT::isVoid)
8064 return 0;
8065 if ((Subtarget->hasFPARMv8())) {
8066 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
8067 }
8068 return 0;
8069}
8070
8071unsigned fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8072 switch (VT.SimpleTy) {
8073 case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
8074 case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
8075 case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
8076 default: return 0;
8077 }
8078}
8079
8080// FastEmit functions for AArch64ISD::FRECPS.
8081
8082unsigned fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8083 if (RetVT.SimpleTy != MVT::f32)
8084 return 0;
8085 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8086}
8087
8088unsigned fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8089 if (RetVT.SimpleTy != MVT::f64)
8090 return 0;
8091 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8092}
8093
8094unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8095 if (RetVT.SimpleTy != MVT::v2f32)
8096 return 0;
8097 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8098}
8099
8100unsigned fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8101 if (RetVT.SimpleTy != MVT::v4f32)
8102 return 0;
8103 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8104}
8105
8106unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8107 if (RetVT.SimpleTy != MVT::v2f64)
8108 return 0;
8109 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8110}
8111
8112unsigned fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8113 if (RetVT.SimpleTy != MVT::nxv8f16)
8114 return 0;
8115 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8116 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8117 }
8118 return 0;
8119}
8120
8121unsigned fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8122 if (RetVT.SimpleTy != MVT::nxv4f32)
8123 return 0;
8124 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8125 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8126 }
8127 return 0;
8128}
8129
8130unsigned fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8131 if (RetVT.SimpleTy != MVT::nxv2f64)
8132 return 0;
8133 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8134 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRECPS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8135 }
8136 return 0;
8137}
8138
8139unsigned fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8140 switch (VT.SimpleTy) {
8141 case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op1);
8142 case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op1);
8143 case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op1);
8144 case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op1);
8145 case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op1);
8146 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRECPS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8147 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRECPS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8148 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRECPS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8149 default: return 0;
8150 }
8151}
8152
8153// FastEmit functions for AArch64ISD::FRSQRTS.
8154
8155unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8156 if (RetVT.SimpleTy != MVT::f32)
8157 return 0;
8158 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS32, RC: &AArch64::FPR32RegClass, Op0, Op1);
8159}
8160
8161unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8162 if (RetVT.SimpleTy != MVT::f64)
8163 return 0;
8164 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS64, RC: &AArch64::FPR64RegClass, Op0, Op1);
8165}
8166
8167unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8168 if (RetVT.SimpleTy != MVT::v2f32)
8169 return 0;
8170 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8171}
8172
8173unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8174 if (RetVT.SimpleTy != MVT::v4f32)
8175 return 0;
8176 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8177}
8178
8179unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8180 if (RetVT.SimpleTy != MVT::v2f64)
8181 return 0;
8182 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTSv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8183}
8184
8185unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8186 if (RetVT.SimpleTy != MVT::nxv8f16)
8187 return 0;
8188 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8189 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8190 }
8191 return 0;
8192}
8193
8194unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8195 if (RetVT.SimpleTy != MVT::nxv4f32)
8196 return 0;
8197 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8198 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8199 }
8200 return 0;
8201}
8202
8203unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8204 if (RetVT.SimpleTy != MVT::nxv2f64)
8205 return 0;
8206 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8207 return fastEmitInst_rr(MachineInstOpcode: AArch64::FRSQRTS_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8208 }
8209 return 0;
8210}
8211
8212unsigned fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8213 switch (VT.SimpleTy) {
8214 case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op1);
8215 case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op1);
8216 case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op1);
8217 case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op1);
8218 case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op1);
8219 case MVT::nxv8f16: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8220 case MVT::nxv4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8221 case MVT::nxv2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8222 default: return 0;
8223 }
8224}
8225
8226// FastEmit functions for AArch64ISD::PMULL.
8227
8228unsigned fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8229 if (RetVT.SimpleTy != MVT::v8i16)
8230 return 0;
8231 if ((Subtarget->isNeonAvailable())) {
8232 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv8i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8233 }
8234 return 0;
8235}
8236
8237unsigned fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8238 if (RetVT.SimpleTy != MVT::v16i8)
8239 return 0;
8240 if ((Subtarget->hasAES())) {
8241 return fastEmitInst_rr(MachineInstOpcode: AArch64::PMULLv1i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8242 }
8243 return 0;
8244}
8245
8246unsigned fastEmit_AArch64ISD_PMULL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8247 switch (VT.SimpleTy) {
8248 case MVT::v8i8: return fastEmit_AArch64ISD_PMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8249 case MVT::v1i64: return fastEmit_AArch64ISD_PMULL_MVT_v1i64_rr(RetVT, Op0, Op1);
8250 default: return 0;
8251 }
8252}
8253
8254// FastEmit functions for AArch64ISD::PTEST.
8255
8256unsigned fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8257 if (RetVT.SimpleTy != MVT::isVoid)
8258 return 0;
8259 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8260 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP, RC: &AArch64::PPRRegClass, Op0, Op1);
8261 }
8262 return 0;
8263}
8264
8265unsigned fastEmit_AArch64ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8266 switch (VT.SimpleTy) {
8267 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8268 default: return 0;
8269 }
8270}
8271
8272// FastEmit functions for AArch64ISD::PTEST_ANY.
8273
8274unsigned fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8275 if (RetVT.SimpleTy != MVT::isVoid)
8276 return 0;
8277 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8278 return fastEmitInst_rr(MachineInstOpcode: AArch64::PTEST_PP_ANY, RC: &AArch64::PPRRegClass, Op0, Op1);
8279 }
8280 return 0;
8281}
8282
8283unsigned fastEmit_AArch64ISD_PTEST_ANY_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8284 switch (VT.SimpleTy) {
8285 case MVT::nxv16i1: return fastEmit_AArch64ISD_PTEST_ANY_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8286 default: return 0;
8287 }
8288}
8289
8290// FastEmit functions for AArch64ISD::SMULL.
8291
8292unsigned fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8293 if (RetVT.SimpleTy != MVT::v8i16)
8294 return 0;
8295 if ((Subtarget->isNeonAvailable())) {
8296 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8297 }
8298 return 0;
8299}
8300
8301unsigned fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8302 if (RetVT.SimpleTy != MVT::v4i32)
8303 return 0;
8304 if ((Subtarget->isNeonAvailable())) {
8305 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8306 }
8307 return 0;
8308}
8309
8310unsigned fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8311 if (RetVT.SimpleTy != MVT::v2i64)
8312 return 0;
8313 if ((Subtarget->isNeonAvailable())) {
8314 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8315 }
8316 return 0;
8317}
8318
8319unsigned fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8320 switch (VT.SimpleTy) {
8321 case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
8322 case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
8323 case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
8324 default: return 0;
8325 }
8326}
8327
8328// FastEmit functions for AArch64ISD::STRICT_FCMP.
8329
8330unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8331 if (RetVT.SimpleTy != MVT::isVoid)
8332 return 0;
8333 if ((Subtarget->hasFullFP16())) {
8334 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
8335 }
8336 return 0;
8337}
8338
8339unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8340 if (RetVT.SimpleTy != MVT::isVoid)
8341 return 0;
8342 if ((Subtarget->hasFPARMv8())) {
8343 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
8344 }
8345 return 0;
8346}
8347
8348unsigned fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8349 if (RetVT.SimpleTy != MVT::isVoid)
8350 return 0;
8351 if ((Subtarget->hasFPARMv8())) {
8352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
8353 }
8354 return 0;
8355}
8356
8357unsigned fastEmit_AArch64ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8358 switch (VT.SimpleTy) {
8359 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f16_rr(RetVT, Op0, Op1);
8360 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op1);
8361 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op1);
8362 default: return 0;
8363 }
8364}
8365
8366// FastEmit functions for AArch64ISD::STRICT_FCMPE.
8367
8368unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8369 if (RetVT.SimpleTy != MVT::isVoid)
8370 return 0;
8371 if ((Subtarget->hasFullFP16())) {
8372 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
8373 }
8374 return 0;
8375}
8376
8377unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8378 if (RetVT.SimpleTy != MVT::isVoid)
8379 return 0;
8380 if ((Subtarget->hasFPARMv8())) {
8381 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPESrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
8382 }
8383 return 0;
8384}
8385
8386unsigned fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8387 if (RetVT.SimpleTy != MVT::isVoid)
8388 return 0;
8389 if ((Subtarget->hasFPARMv8())) {
8390 return fastEmitInst_rr(MachineInstOpcode: AArch64::FCMPEDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
8391 }
8392 return 0;
8393}
8394
8395unsigned fastEmit_AArch64ISD_STRICT_FCMPE_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8396 switch (VT.SimpleTy) {
8397 case MVT::f16: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f16_rr(RetVT, Op0, Op1);
8398 case MVT::f32: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f32_rr(RetVT, Op0, Op1);
8399 case MVT::f64: return fastEmit_AArch64ISD_STRICT_FCMPE_MVT_f64_rr(RetVT, Op0, Op1);
8400 default: return 0;
8401 }
8402}
8403
8404// FastEmit functions for AArch64ISD::TBL.
8405
8406unsigned fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8407 if (RetVT.SimpleTy != MVT::nxv16i8)
8408 return 0;
8409 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8410 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8411 }
8412 return 0;
8413}
8414
8415unsigned fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8416 if (RetVT.SimpleTy != MVT::nxv8i16)
8417 return 0;
8418 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8419 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8420 }
8421 return 0;
8422}
8423
8424unsigned fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8425 if (RetVT.SimpleTy != MVT::nxv4i32)
8426 return 0;
8427 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8428 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8429 }
8430 return 0;
8431}
8432
8433unsigned fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8434 if (RetVT.SimpleTy != MVT::nxv2i64)
8435 return 0;
8436 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8437 return fastEmitInst_rr(MachineInstOpcode: AArch64::TBL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8438 }
8439 return 0;
8440}
8441
8442unsigned fastEmit_AArch64ISD_TBL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8443 switch (VT.SimpleTy) {
8444 case MVT::nxv16i8: return fastEmit_AArch64ISD_TBL_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8445 case MVT::nxv8i16: return fastEmit_AArch64ISD_TBL_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8446 case MVT::nxv4i32: return fastEmit_AArch64ISD_TBL_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8447 case MVT::nxv2i64: return fastEmit_AArch64ISD_TBL_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8448 default: return 0;
8449 }
8450}
8451
8452// FastEmit functions for AArch64ISD::TRN1.
8453
8454unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8455 if (RetVT.SimpleTy != MVT::v8i8)
8456 return 0;
8457 if ((Subtarget->isNeonAvailable())) {
8458 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8459 }
8460 return 0;
8461}
8462
8463unsigned fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8464 if (RetVT.SimpleTy != MVT::v16i8)
8465 return 0;
8466 if ((Subtarget->isNeonAvailable())) {
8467 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8468 }
8469 return 0;
8470}
8471
8472unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8473 if (RetVT.SimpleTy != MVT::v4i16)
8474 return 0;
8475 if ((Subtarget->isNeonAvailable())) {
8476 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8477 }
8478 return 0;
8479}
8480
8481unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8482 if (RetVT.SimpleTy != MVT::v8i16)
8483 return 0;
8484 if ((Subtarget->isNeonAvailable())) {
8485 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8486 }
8487 return 0;
8488}
8489
8490unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8491 if (RetVT.SimpleTy != MVT::v2i32)
8492 return 0;
8493 if ((Subtarget->isNeonAvailable())) {
8494 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8495 }
8496 return 0;
8497}
8498
8499unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8500 if (RetVT.SimpleTy != MVT::v4i32)
8501 return 0;
8502 if ((Subtarget->isNeonAvailable())) {
8503 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8504 }
8505 return 0;
8506}
8507
8508unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8509 if (RetVT.SimpleTy != MVT::v2i64)
8510 return 0;
8511 if ((Subtarget->isNeonAvailable())) {
8512 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8513 }
8514 return 0;
8515}
8516
8517unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8518 if (RetVT.SimpleTy != MVT::v4f16)
8519 return 0;
8520 if ((Subtarget->isNeonAvailable())) {
8521 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8522 }
8523 return 0;
8524}
8525
8526unsigned fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8527 if (RetVT.SimpleTy != MVT::v8f16)
8528 return 0;
8529 if ((Subtarget->isNeonAvailable())) {
8530 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8531 }
8532 return 0;
8533}
8534
8535unsigned fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8536 if (RetVT.SimpleTy != MVT::v4bf16)
8537 return 0;
8538 if ((Subtarget->isNeonAvailable())) {
8539 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8540 }
8541 return 0;
8542}
8543
8544unsigned fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8545 if (RetVT.SimpleTy != MVT::v8bf16)
8546 return 0;
8547 if ((Subtarget->isNeonAvailable())) {
8548 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8549 }
8550 return 0;
8551}
8552
8553unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8554 if (RetVT.SimpleTy != MVT::v2f32)
8555 return 0;
8556 if ((Subtarget->isNeonAvailable())) {
8557 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8558 }
8559 return 0;
8560}
8561
8562unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8563 if (RetVT.SimpleTy != MVT::v4f32)
8564 return 0;
8565 if ((Subtarget->isNeonAvailable())) {
8566 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8567 }
8568 return 0;
8569}
8570
8571unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8572 if (RetVT.SimpleTy != MVT::v2f64)
8573 return 0;
8574 if ((Subtarget->isNeonAvailable())) {
8575 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8576 }
8577 return 0;
8578}
8579
8580unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8581 if (RetVT.SimpleTy != MVT::nxv2i1)
8582 return 0;
8583 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8584 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8585 }
8586 return 0;
8587}
8588
8589unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8590 if (RetVT.SimpleTy != MVT::nxv4i1)
8591 return 0;
8592 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8593 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8594 }
8595 return 0;
8596}
8597
8598unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8599 if (RetVT.SimpleTy != MVT::nxv8i1)
8600 return 0;
8601 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8602 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8603 }
8604 return 0;
8605}
8606
8607unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8608 if (RetVT.SimpleTy != MVT::nxv16i1)
8609 return 0;
8610 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8611 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8612 }
8613 return 0;
8614}
8615
8616unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8617 if (RetVT.SimpleTy != MVT::nxv16i8)
8618 return 0;
8619 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8620 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8621 }
8622 return 0;
8623}
8624
8625unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8626 if (RetVT.SimpleTy != MVT::nxv8i16)
8627 return 0;
8628 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8629 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8630 }
8631 return 0;
8632}
8633
8634unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8635 if (RetVT.SimpleTy != MVT::nxv4i32)
8636 return 0;
8637 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8638 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8639 }
8640 return 0;
8641}
8642
8643unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8644 if (RetVT.SimpleTy != MVT::nxv2i64)
8645 return 0;
8646 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8647 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8648 }
8649 return 0;
8650}
8651
8652unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8653 if (RetVT.SimpleTy != MVT::nxv2f16)
8654 return 0;
8655 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8656 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8657 }
8658 return 0;
8659}
8660
8661unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8662 if (RetVT.SimpleTy != MVT::nxv4f16)
8663 return 0;
8664 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8665 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8666 }
8667 return 0;
8668}
8669
8670unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8671 if (RetVT.SimpleTy != MVT::nxv8f16)
8672 return 0;
8673 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8674 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8675 }
8676 return 0;
8677}
8678
8679unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8680 if (RetVT.SimpleTy != MVT::nxv8bf16)
8681 return 0;
8682 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8683 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8684 }
8685 return 0;
8686}
8687
8688unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8689 if (RetVT.SimpleTy != MVT::nxv2f32)
8690 return 0;
8691 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8692 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8693 }
8694 return 0;
8695}
8696
8697unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8698 if (RetVT.SimpleTy != MVT::nxv4f32)
8699 return 0;
8700 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8701 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8702 }
8703 return 0;
8704}
8705
8706unsigned fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8707 if (RetVT.SimpleTy != MVT::nxv2f64)
8708 return 0;
8709 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8710 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8711 }
8712 return 0;
8713}
8714
8715unsigned fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
8716 switch (VT.SimpleTy) {
8717 case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op1);
8718 case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op1);
8719 case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op1);
8720 case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op1);
8721 case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op1);
8722 case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op1);
8723 case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op1);
8724 case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op1);
8725 case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op1);
8726 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN1_MVT_v4bf16_rr(RetVT, Op0, Op1);
8727 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN1_MVT_v8bf16_rr(RetVT, Op0, Op1);
8728 case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op1);
8729 case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op1);
8730 case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op1);
8731 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
8732 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
8733 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
8734 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
8735 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
8736 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
8737 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
8738 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
8739 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
8740 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
8741 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
8742 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
8743 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
8744 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
8745 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
8746 default: return 0;
8747 }
8748}
8749
8750// FastEmit functions for AArch64ISD::TRN2.
8751
8752unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8753 if (RetVT.SimpleTy != MVT::v8i8)
8754 return 0;
8755 if ((Subtarget->isNeonAvailable())) {
8756 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
8757 }
8758 return 0;
8759}
8760
8761unsigned fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8762 if (RetVT.SimpleTy != MVT::v16i8)
8763 return 0;
8764 if ((Subtarget->isNeonAvailable())) {
8765 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
8766 }
8767 return 0;
8768}
8769
8770unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8771 if (RetVT.SimpleTy != MVT::v4i16)
8772 return 0;
8773 if ((Subtarget->isNeonAvailable())) {
8774 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8775 }
8776 return 0;
8777}
8778
8779unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8780 if (RetVT.SimpleTy != MVT::v8i16)
8781 return 0;
8782 if ((Subtarget->isNeonAvailable())) {
8783 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8784 }
8785 return 0;
8786}
8787
8788unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8789 if (RetVT.SimpleTy != MVT::v2i32)
8790 return 0;
8791 if ((Subtarget->isNeonAvailable())) {
8792 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8793 }
8794 return 0;
8795}
8796
8797unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8798 if (RetVT.SimpleTy != MVT::v4i32)
8799 return 0;
8800 if ((Subtarget->isNeonAvailable())) {
8801 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8802 }
8803 return 0;
8804}
8805
8806unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8807 if (RetVT.SimpleTy != MVT::v2i64)
8808 return 0;
8809 if ((Subtarget->isNeonAvailable())) {
8810 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8811 }
8812 return 0;
8813}
8814
8815unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8816 if (RetVT.SimpleTy != MVT::v4f16)
8817 return 0;
8818 if ((Subtarget->isNeonAvailable())) {
8819 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8820 }
8821 return 0;
8822}
8823
8824unsigned fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8825 if (RetVT.SimpleTy != MVT::v8f16)
8826 return 0;
8827 if ((Subtarget->isNeonAvailable())) {
8828 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8829 }
8830 return 0;
8831}
8832
8833unsigned fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8834 if (RetVT.SimpleTy != MVT::v4bf16)
8835 return 0;
8836 if ((Subtarget->isNeonAvailable())) {
8837 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
8838 }
8839 return 0;
8840}
8841
8842unsigned fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8843 if (RetVT.SimpleTy != MVT::v8bf16)
8844 return 0;
8845 if ((Subtarget->isNeonAvailable())) {
8846 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
8847 }
8848 return 0;
8849}
8850
8851unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8852 if (RetVT.SimpleTy != MVT::v2f32)
8853 return 0;
8854 if ((Subtarget->isNeonAvailable())) {
8855 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
8856 }
8857 return 0;
8858}
8859
8860unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8861 if (RetVT.SimpleTy != MVT::v4f32)
8862 return 0;
8863 if ((Subtarget->isNeonAvailable())) {
8864 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
8865 }
8866 return 0;
8867}
8868
8869unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8870 if (RetVT.SimpleTy != MVT::v2f64)
8871 return 0;
8872 if ((Subtarget->isNeonAvailable())) {
8873 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
8874 }
8875 return 0;
8876}
8877
8878unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8879 if (RetVT.SimpleTy != MVT::nxv2i1)
8880 return 0;
8881 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8882 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
8883 }
8884 return 0;
8885}
8886
8887unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8888 if (RetVT.SimpleTy != MVT::nxv4i1)
8889 return 0;
8890 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8891 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
8892 }
8893 return 0;
8894}
8895
8896unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8897 if (RetVT.SimpleTy != MVT::nxv8i1)
8898 return 0;
8899 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8900 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
8901 }
8902 return 0;
8903}
8904
8905unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8906 if (RetVT.SimpleTy != MVT::nxv16i1)
8907 return 0;
8908 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8909 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
8910 }
8911 return 0;
8912}
8913
8914unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8915 if (RetVT.SimpleTy != MVT::nxv16i8)
8916 return 0;
8917 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8918 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
8919 }
8920 return 0;
8921}
8922
8923unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8924 if (RetVT.SimpleTy != MVT::nxv8i16)
8925 return 0;
8926 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8927 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8928 }
8929 return 0;
8930}
8931
8932unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8933 if (RetVT.SimpleTy != MVT::nxv4i32)
8934 return 0;
8935 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8936 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8937 }
8938 return 0;
8939}
8940
8941unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8942 if (RetVT.SimpleTy != MVT::nxv2i64)
8943 return 0;
8944 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8945 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8946 }
8947 return 0;
8948}
8949
8950unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8951 if (RetVT.SimpleTy != MVT::nxv2f16)
8952 return 0;
8953 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8954 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8955 }
8956 return 0;
8957}
8958
8959unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8960 if (RetVT.SimpleTy != MVT::nxv4f16)
8961 return 0;
8962 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8963 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
8964 }
8965 return 0;
8966}
8967
8968unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8969 if (RetVT.SimpleTy != MVT::nxv8f16)
8970 return 0;
8971 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8972 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8973 }
8974 return 0;
8975}
8976
8977unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8978 if (RetVT.SimpleTy != MVT::nxv8bf16)
8979 return 0;
8980 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8981 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
8982 }
8983 return 0;
8984}
8985
8986unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8987 if (RetVT.SimpleTy != MVT::nxv2f32)
8988 return 0;
8989 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8990 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
8991 }
8992 return 0;
8993}
8994
8995unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
8996 if (RetVT.SimpleTy != MVT::nxv4f32)
8997 return 0;
8998 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
8999 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9000 }
9001 return 0;
9002}
9003
9004unsigned fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9005 if (RetVT.SimpleTy != MVT::nxv2f64)
9006 return 0;
9007 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9008 return fastEmitInst_rr(MachineInstOpcode: AArch64::TRN2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9009 }
9010 return 0;
9011}
9012
9013unsigned fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9014 switch (VT.SimpleTy) {
9015 case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op1);
9016 case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op1);
9017 case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op1);
9018 case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op1);
9019 case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op1);
9020 case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op1);
9021 case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op1);
9022 case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op1);
9023 case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op1);
9024 case MVT::v4bf16: return fastEmit_AArch64ISD_TRN2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9025 case MVT::v8bf16: return fastEmit_AArch64ISD_TRN2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9026 case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op1);
9027 case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op1);
9028 case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op1);
9029 case MVT::nxv2i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9030 case MVT::nxv4i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9031 case MVT::nxv8i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9032 case MVT::nxv16i1: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9033 case MVT::nxv16i8: return fastEmit_AArch64ISD_TRN2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9034 case MVT::nxv8i16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9035 case MVT::nxv4i32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9036 case MVT::nxv2i64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9037 case MVT::nxv2f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9038 case MVT::nxv4f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9039 case MVT::nxv8f16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9040 case MVT::nxv8bf16: return fastEmit_AArch64ISD_TRN2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9041 case MVT::nxv2f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9042 case MVT::nxv4f32: return fastEmit_AArch64ISD_TRN2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9043 case MVT::nxv2f64: return fastEmit_AArch64ISD_TRN2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9044 default: return 0;
9045 }
9046}
9047
9048// FastEmit functions for AArch64ISD::UMULL.
9049
9050unsigned fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9051 if (RetVT.SimpleTy != MVT::v8i16)
9052 return 0;
9053 if ((Subtarget->isNeonAvailable())) {
9054 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv8i8_v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9055 }
9056 return 0;
9057}
9058
9059unsigned fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9060 if (RetVT.SimpleTy != MVT::v4i32)
9061 return 0;
9062 if ((Subtarget->isNeonAvailable())) {
9063 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv4i16_v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9064 }
9065 return 0;
9066}
9067
9068unsigned fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9069 if (RetVT.SimpleTy != MVT::v2i64)
9070 return 0;
9071 if ((Subtarget->isNeonAvailable())) {
9072 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULLv2i32_v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9073 }
9074 return 0;
9075}
9076
9077unsigned fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9078 switch (VT.SimpleTy) {
9079 case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op1);
9080 case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op1);
9081 case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op1);
9082 default: return 0;
9083 }
9084}
9085
9086// FastEmit functions for AArch64ISD::UZP1.
9087
9088unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9089 if (RetVT.SimpleTy != MVT::v8i8)
9090 return 0;
9091 if ((Subtarget->isNeonAvailable())) {
9092 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9093 }
9094 return 0;
9095}
9096
9097unsigned fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9098 if (RetVT.SimpleTy != MVT::v16i8)
9099 return 0;
9100 if ((Subtarget->isNeonAvailable())) {
9101 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9102 }
9103 return 0;
9104}
9105
9106unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9107 if (RetVT.SimpleTy != MVT::v4i16)
9108 return 0;
9109 if ((Subtarget->isNeonAvailable())) {
9110 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9111 }
9112 return 0;
9113}
9114
9115unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9116 if (RetVT.SimpleTy != MVT::v8i16)
9117 return 0;
9118 if ((Subtarget->isNeonAvailable())) {
9119 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9120 }
9121 return 0;
9122}
9123
9124unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9125 if (RetVT.SimpleTy != MVT::v2i32)
9126 return 0;
9127 if ((Subtarget->isNeonAvailable())) {
9128 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9129 }
9130 return 0;
9131}
9132
9133unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9134 if (RetVT.SimpleTy != MVT::v4i32)
9135 return 0;
9136 if ((Subtarget->isNeonAvailable())) {
9137 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9138 }
9139 return 0;
9140}
9141
9142unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9143 if (RetVT.SimpleTy != MVT::v2i64)
9144 return 0;
9145 if ((Subtarget->isNeonAvailable())) {
9146 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9147 }
9148 return 0;
9149}
9150
9151unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9152 if (RetVT.SimpleTy != MVT::v4f16)
9153 return 0;
9154 if ((Subtarget->isNeonAvailable())) {
9155 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9156 }
9157 return 0;
9158}
9159
9160unsigned fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9161 if (RetVT.SimpleTy != MVT::v8f16)
9162 return 0;
9163 if ((Subtarget->isNeonAvailable())) {
9164 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9165 }
9166 return 0;
9167}
9168
9169unsigned fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9170 if (RetVT.SimpleTy != MVT::v4bf16)
9171 return 0;
9172 if ((Subtarget->isNeonAvailable())) {
9173 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9174 }
9175 return 0;
9176}
9177
9178unsigned fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9179 if (RetVT.SimpleTy != MVT::v8bf16)
9180 return 0;
9181 if ((Subtarget->isNeonAvailable())) {
9182 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9183 }
9184 return 0;
9185}
9186
9187unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9188 if (RetVT.SimpleTy != MVT::v2f32)
9189 return 0;
9190 if ((Subtarget->isNeonAvailable())) {
9191 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9192 }
9193 return 0;
9194}
9195
9196unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9197 if (RetVT.SimpleTy != MVT::v4f32)
9198 return 0;
9199 if ((Subtarget->isNeonAvailable())) {
9200 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9201 }
9202 return 0;
9203}
9204
9205unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9206 if (RetVT.SimpleTy != MVT::v2f64)
9207 return 0;
9208 if ((Subtarget->isNeonAvailable())) {
9209 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9210 }
9211 return 0;
9212}
9213
9214unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9215 if (RetVT.SimpleTy != MVT::nxv2i1)
9216 return 0;
9217 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9218 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9219 }
9220 return 0;
9221}
9222
9223unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9224 if (RetVT.SimpleTy != MVT::nxv4i1)
9225 return 0;
9226 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9227 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9228 }
9229 return 0;
9230}
9231
9232unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9233 if (RetVT.SimpleTy != MVT::nxv8i1)
9234 return 0;
9235 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9236 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9237 }
9238 return 0;
9239}
9240
9241unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9242 if (RetVT.SimpleTy != MVT::nxv16i1)
9243 return 0;
9244 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9245 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9246 }
9247 return 0;
9248}
9249
9250unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9251 if (RetVT.SimpleTy != MVT::nxv16i8)
9252 return 0;
9253 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9254 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9255 }
9256 return 0;
9257}
9258
9259unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9260 if (RetVT.SimpleTy != MVT::nxv8i16)
9261 return 0;
9262 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9263 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9264 }
9265 return 0;
9266}
9267
9268unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9269 if (RetVT.SimpleTy != MVT::nxv4i32)
9270 return 0;
9271 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9272 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9273 }
9274 return 0;
9275}
9276
9277unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9278 if (RetVT.SimpleTy != MVT::nxv2i64)
9279 return 0;
9280 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9281 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9282 }
9283 return 0;
9284}
9285
9286unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9287 if (RetVT.SimpleTy != MVT::nxv2f16)
9288 return 0;
9289 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9290 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9291 }
9292 return 0;
9293}
9294
9295unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9296 if (RetVT.SimpleTy != MVT::nxv4f16)
9297 return 0;
9298 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9299 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9300 }
9301 return 0;
9302}
9303
9304unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9305 if (RetVT.SimpleTy != MVT::nxv8f16)
9306 return 0;
9307 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9308 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9309 }
9310 return 0;
9311}
9312
9313unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9314 if (RetVT.SimpleTy != MVT::nxv8bf16)
9315 return 0;
9316 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9317 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9318 }
9319 return 0;
9320}
9321
9322unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9323 if (RetVT.SimpleTy != MVT::nxv2f32)
9324 return 0;
9325 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9326 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9327 }
9328 return 0;
9329}
9330
9331unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9332 if (RetVT.SimpleTy != MVT::nxv4f32)
9333 return 0;
9334 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9335 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9336 }
9337 return 0;
9338}
9339
9340unsigned fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9341 if (RetVT.SimpleTy != MVT::nxv2f64)
9342 return 0;
9343 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9344 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9345 }
9346 return 0;
9347}
9348
9349unsigned fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9350 switch (VT.SimpleTy) {
9351 case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9352 case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9353 case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9354 case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9355 case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9356 case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9357 case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9358 case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9359 case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9360 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9361 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9362 case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9363 case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9364 case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9365 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9366 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9367 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9368 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9369 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9370 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9371 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9372 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9373 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9374 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9375 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9376 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9377 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9378 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9379 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9380 default: return 0;
9381 }
9382}
9383
9384// FastEmit functions for AArch64ISD::UZP2.
9385
9386unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9387 if (RetVT.SimpleTy != MVT::v8i8)
9388 return 0;
9389 if ((Subtarget->isNeonAvailable())) {
9390 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9391 }
9392 return 0;
9393}
9394
9395unsigned fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9396 if (RetVT.SimpleTy != MVT::v16i8)
9397 return 0;
9398 if ((Subtarget->isNeonAvailable())) {
9399 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9400 }
9401 return 0;
9402}
9403
9404unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9405 if (RetVT.SimpleTy != MVT::v4i16)
9406 return 0;
9407 if ((Subtarget->isNeonAvailable())) {
9408 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9409 }
9410 return 0;
9411}
9412
9413unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9414 if (RetVT.SimpleTy != MVT::v8i16)
9415 return 0;
9416 if ((Subtarget->isNeonAvailable())) {
9417 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9418 }
9419 return 0;
9420}
9421
9422unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9423 if (RetVT.SimpleTy != MVT::v2i32)
9424 return 0;
9425 if ((Subtarget->isNeonAvailable())) {
9426 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9427 }
9428 return 0;
9429}
9430
9431unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9432 if (RetVT.SimpleTy != MVT::v4i32)
9433 return 0;
9434 if ((Subtarget->isNeonAvailable())) {
9435 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9436 }
9437 return 0;
9438}
9439
9440unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9441 if (RetVT.SimpleTy != MVT::v2i64)
9442 return 0;
9443 if ((Subtarget->isNeonAvailable())) {
9444 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9445 }
9446 return 0;
9447}
9448
9449unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9450 if (RetVT.SimpleTy != MVT::v4f16)
9451 return 0;
9452 if ((Subtarget->isNeonAvailable())) {
9453 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9454 }
9455 return 0;
9456}
9457
9458unsigned fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9459 if (RetVT.SimpleTy != MVT::v8f16)
9460 return 0;
9461 if ((Subtarget->isNeonAvailable())) {
9462 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9463 }
9464 return 0;
9465}
9466
9467unsigned fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9468 if (RetVT.SimpleTy != MVT::v4bf16)
9469 return 0;
9470 if ((Subtarget->isNeonAvailable())) {
9471 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9472 }
9473 return 0;
9474}
9475
9476unsigned fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9477 if (RetVT.SimpleTy != MVT::v8bf16)
9478 return 0;
9479 if ((Subtarget->isNeonAvailable())) {
9480 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9481 }
9482 return 0;
9483}
9484
9485unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9486 if (RetVT.SimpleTy != MVT::v2f32)
9487 return 0;
9488 if ((Subtarget->isNeonAvailable())) {
9489 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9490 }
9491 return 0;
9492}
9493
9494unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9495 if (RetVT.SimpleTy != MVT::v4f32)
9496 return 0;
9497 if ((Subtarget->isNeonAvailable())) {
9498 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9499 }
9500 return 0;
9501}
9502
9503unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9504 if (RetVT.SimpleTy != MVT::v2f64)
9505 return 0;
9506 if ((Subtarget->isNeonAvailable())) {
9507 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9508 }
9509 return 0;
9510}
9511
9512unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9513 if (RetVT.SimpleTy != MVT::nxv2i1)
9514 return 0;
9515 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9516 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9517 }
9518 return 0;
9519}
9520
9521unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9522 if (RetVT.SimpleTy != MVT::nxv4i1)
9523 return 0;
9524 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9525 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9526 }
9527 return 0;
9528}
9529
9530unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9531 if (RetVT.SimpleTy != MVT::nxv8i1)
9532 return 0;
9533 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9534 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9535 }
9536 return 0;
9537}
9538
9539unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9540 if (RetVT.SimpleTy != MVT::nxv16i1)
9541 return 0;
9542 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9543 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9544 }
9545 return 0;
9546}
9547
9548unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9549 if (RetVT.SimpleTy != MVT::nxv16i8)
9550 return 0;
9551 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9552 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9553 }
9554 return 0;
9555}
9556
9557unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9558 if (RetVT.SimpleTy != MVT::nxv8i16)
9559 return 0;
9560 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9561 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9562 }
9563 return 0;
9564}
9565
9566unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9567 if (RetVT.SimpleTy != MVT::nxv4i32)
9568 return 0;
9569 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9570 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9571 }
9572 return 0;
9573}
9574
9575unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9576 if (RetVT.SimpleTy != MVT::nxv2i64)
9577 return 0;
9578 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9579 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9580 }
9581 return 0;
9582}
9583
9584unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9585 if (RetVT.SimpleTy != MVT::nxv2f16)
9586 return 0;
9587 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9588 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9589 }
9590 return 0;
9591}
9592
9593unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9594 if (RetVT.SimpleTy != MVT::nxv4f16)
9595 return 0;
9596 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9597 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9598 }
9599 return 0;
9600}
9601
9602unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9603 if (RetVT.SimpleTy != MVT::nxv8f16)
9604 return 0;
9605 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9606 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9607 }
9608 return 0;
9609}
9610
9611unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9612 if (RetVT.SimpleTy != MVT::nxv8bf16)
9613 return 0;
9614 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9615 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9616 }
9617 return 0;
9618}
9619
9620unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9621 if (RetVT.SimpleTy != MVT::nxv2f32)
9622 return 0;
9623 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9624 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9625 }
9626 return 0;
9627}
9628
9629unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9630 if (RetVT.SimpleTy != MVT::nxv4f32)
9631 return 0;
9632 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9633 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9634 }
9635 return 0;
9636}
9637
9638unsigned fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9639 if (RetVT.SimpleTy != MVT::nxv2f64)
9640 return 0;
9641 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9642 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9643 }
9644 return 0;
9645}
9646
9647unsigned fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9648 switch (VT.SimpleTy) {
9649 case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op1);
9650 case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op1);
9651 case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op1);
9652 case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op1);
9653 case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op1);
9654 case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op1);
9655 case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op1);
9656 case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op1);
9657 case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op1);
9658 case MVT::v4bf16: return fastEmit_AArch64ISD_UZP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
9659 case MVT::v8bf16: return fastEmit_AArch64ISD_UZP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
9660 case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op1);
9661 case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op1);
9662 case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op1);
9663 case MVT::nxv2i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9664 case MVT::nxv4i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9665 case MVT::nxv8i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9666 case MVT::nxv16i1: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9667 case MVT::nxv16i8: return fastEmit_AArch64ISD_UZP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9668 case MVT::nxv8i16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9669 case MVT::nxv4i32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9670 case MVT::nxv2i64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9671 case MVT::nxv2f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9672 case MVT::nxv4f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9673 case MVT::nxv8f16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9674 case MVT::nxv8bf16: return fastEmit_AArch64ISD_UZP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9675 case MVT::nxv2f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9676 case MVT::nxv4f32: return fastEmit_AArch64ISD_UZP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9677 case MVT::nxv2f64: return fastEmit_AArch64ISD_UZP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9678 default: return 0;
9679 }
9680}
9681
9682// FastEmit functions for AArch64ISD::ZIP1.
9683
9684unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9685 if (RetVT.SimpleTy != MVT::v8i8)
9686 return 0;
9687 if ((Subtarget->isNeonAvailable())) {
9688 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9689 }
9690 return 0;
9691}
9692
9693unsigned fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9694 if (RetVT.SimpleTy != MVT::v16i8)
9695 return 0;
9696 if ((Subtarget->isNeonAvailable())) {
9697 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9698 }
9699 return 0;
9700}
9701
9702unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9703 if (RetVT.SimpleTy != MVT::v4i16)
9704 return 0;
9705 if ((Subtarget->isNeonAvailable())) {
9706 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9707 }
9708 return 0;
9709}
9710
9711unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9712 if (RetVT.SimpleTy != MVT::v8i16)
9713 return 0;
9714 if ((Subtarget->isNeonAvailable())) {
9715 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9716 }
9717 return 0;
9718}
9719
9720unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9721 if (RetVT.SimpleTy != MVT::v2i32)
9722 return 0;
9723 if ((Subtarget->isNeonAvailable())) {
9724 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9725 }
9726 return 0;
9727}
9728
9729unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9730 if (RetVT.SimpleTy != MVT::v4i32)
9731 return 0;
9732 if ((Subtarget->isNeonAvailable())) {
9733 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9734 }
9735 return 0;
9736}
9737
9738unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9739 if (RetVT.SimpleTy != MVT::v2i64)
9740 return 0;
9741 if ((Subtarget->isNeonAvailable())) {
9742 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9743 }
9744 return 0;
9745}
9746
9747unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9748 if (RetVT.SimpleTy != MVT::v4f16)
9749 return 0;
9750 if ((Subtarget->isNeonAvailable())) {
9751 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9752 }
9753 return 0;
9754}
9755
9756unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9757 if (RetVT.SimpleTy != MVT::v8f16)
9758 return 0;
9759 if ((Subtarget->isNeonAvailable())) {
9760 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9761 }
9762 return 0;
9763}
9764
9765unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9766 if (RetVT.SimpleTy != MVT::v4bf16)
9767 return 0;
9768 if ((Subtarget->isNeonAvailable())) {
9769 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
9770 }
9771 return 0;
9772}
9773
9774unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9775 if (RetVT.SimpleTy != MVT::v8bf16)
9776 return 0;
9777 if ((Subtarget->isNeonAvailable())) {
9778 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
9779 }
9780 return 0;
9781}
9782
9783unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9784 if (RetVT.SimpleTy != MVT::v2f32)
9785 return 0;
9786 if ((Subtarget->isNeonAvailable())) {
9787 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
9788 }
9789 return 0;
9790}
9791
9792unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9793 if (RetVT.SimpleTy != MVT::v4f32)
9794 return 0;
9795 if ((Subtarget->isNeonAvailable())) {
9796 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
9797 }
9798 return 0;
9799}
9800
9801unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9802 if (RetVT.SimpleTy != MVT::v2f64)
9803 return 0;
9804 if ((Subtarget->isNeonAvailable())) {
9805 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
9806 }
9807 return 0;
9808}
9809
9810unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9811 if (RetVT.SimpleTy != MVT::nxv2i1)
9812 return 0;
9813 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9814 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
9815 }
9816 return 0;
9817}
9818
9819unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9820 if (RetVT.SimpleTy != MVT::nxv4i1)
9821 return 0;
9822 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9823 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
9824 }
9825 return 0;
9826}
9827
9828unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9829 if (RetVT.SimpleTy != MVT::nxv8i1)
9830 return 0;
9831 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9832 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
9833 }
9834 return 0;
9835}
9836
9837unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9838 if (RetVT.SimpleTy != MVT::nxv16i1)
9839 return 0;
9840 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9841 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
9842 }
9843 return 0;
9844}
9845
9846unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9847 if (RetVT.SimpleTy != MVT::nxv16i8)
9848 return 0;
9849 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9850 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
9851 }
9852 return 0;
9853}
9854
9855unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9856 if (RetVT.SimpleTy != MVT::nxv8i16)
9857 return 0;
9858 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9859 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9860 }
9861 return 0;
9862}
9863
9864unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9865 if (RetVT.SimpleTy != MVT::nxv4i32)
9866 return 0;
9867 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9868 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9869 }
9870 return 0;
9871}
9872
9873unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9874 if (RetVT.SimpleTy != MVT::nxv2i64)
9875 return 0;
9876 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9877 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9878 }
9879 return 0;
9880}
9881
9882unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9883 if (RetVT.SimpleTy != MVT::nxv2f16)
9884 return 0;
9885 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9886 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9887 }
9888 return 0;
9889}
9890
9891unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9892 if (RetVT.SimpleTy != MVT::nxv4f16)
9893 return 0;
9894 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9895 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9896 }
9897 return 0;
9898}
9899
9900unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9901 if (RetVT.SimpleTy != MVT::nxv8f16)
9902 return 0;
9903 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9904 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9905 }
9906 return 0;
9907}
9908
9909unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9910 if (RetVT.SimpleTy != MVT::nxv8bf16)
9911 return 0;
9912 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9913 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
9914 }
9915 return 0;
9916}
9917
9918unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9919 if (RetVT.SimpleTy != MVT::nxv2f32)
9920 return 0;
9921 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9922 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9923 }
9924 return 0;
9925}
9926
9927unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9928 if (RetVT.SimpleTy != MVT::nxv4f32)
9929 return 0;
9930 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9931 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
9932 }
9933 return 0;
9934}
9935
9936unsigned fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9937 if (RetVT.SimpleTy != MVT::nxv2f64)
9938 return 0;
9939 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
9940 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP1_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
9941 }
9942 return 0;
9943}
9944
9945unsigned fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
9946 switch (VT.SimpleTy) {
9947 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op1);
9948 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op1);
9949 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op1);
9950 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op1);
9951 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op1);
9952 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op1);
9953 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op1);
9954 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op1);
9955 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op1);
9956 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v4bf16_rr(RetVT, Op0, Op1);
9957 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_v8bf16_rr(RetVT, Op0, Op1);
9958 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op1);
9959 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op1);
9960 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op1);
9961 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i1_rr(RetVT, Op0, Op1);
9962 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i1_rr(RetVT, Op0, Op1);
9963 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i1_rr(RetVT, Op0, Op1);
9964 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i1_rr(RetVT, Op0, Op1);
9965 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP1_MVT_nxv16i8_rr(RetVT, Op0, Op1);
9966 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8i16_rr(RetVT, Op0, Op1);
9967 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4i32_rr(RetVT, Op0, Op1);
9968 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2i64_rr(RetVT, Op0, Op1);
9969 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f16_rr(RetVT, Op0, Op1);
9970 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f16_rr(RetVT, Op0, Op1);
9971 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8f16_rr(RetVT, Op0, Op1);
9972 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP1_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
9973 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f32_rr(RetVT, Op0, Op1);
9974 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP1_MVT_nxv4f32_rr(RetVT, Op0, Op1);
9975 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP1_MVT_nxv2f64_rr(RetVT, Op0, Op1);
9976 default: return 0;
9977 }
9978}
9979
9980// FastEmit functions for AArch64ISD::ZIP2.
9981
9982unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9983 if (RetVT.SimpleTy != MVT::v8i8)
9984 return 0;
9985 if ((Subtarget->isNeonAvailable())) {
9986 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
9987 }
9988 return 0;
9989}
9990
9991unsigned fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
9992 if (RetVT.SimpleTy != MVT::v16i8)
9993 return 0;
9994 if ((Subtarget->isNeonAvailable())) {
9995 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
9996 }
9997 return 0;
9998}
9999
10000unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10001 if (RetVT.SimpleTy != MVT::v4i16)
10002 return 0;
10003 if ((Subtarget->isNeonAvailable())) {
10004 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10005 }
10006 return 0;
10007}
10008
10009unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10010 if (RetVT.SimpleTy != MVT::v8i16)
10011 return 0;
10012 if ((Subtarget->isNeonAvailable())) {
10013 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10014 }
10015 return 0;
10016}
10017
10018unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10019 if (RetVT.SimpleTy != MVT::v2i32)
10020 return 0;
10021 if ((Subtarget->isNeonAvailable())) {
10022 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10023 }
10024 return 0;
10025}
10026
10027unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10028 if (RetVT.SimpleTy != MVT::v4i32)
10029 return 0;
10030 if ((Subtarget->isNeonAvailable())) {
10031 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10032 }
10033 return 0;
10034}
10035
10036unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10037 if (RetVT.SimpleTy != MVT::v2i64)
10038 return 0;
10039 if ((Subtarget->isNeonAvailable())) {
10040 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10041 }
10042 return 0;
10043}
10044
10045unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10046 if (RetVT.SimpleTy != MVT::v4f16)
10047 return 0;
10048 if ((Subtarget->isNeonAvailable())) {
10049 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10050 }
10051 return 0;
10052}
10053
10054unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10055 if (RetVT.SimpleTy != MVT::v8f16)
10056 return 0;
10057 if ((Subtarget->isNeonAvailable())) {
10058 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10059 }
10060 return 0;
10061}
10062
10063unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10064 if (RetVT.SimpleTy != MVT::v4bf16)
10065 return 0;
10066 if ((Subtarget->isNeonAvailable())) {
10067 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10068 }
10069 return 0;
10070}
10071
10072unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10073 if (RetVT.SimpleTy != MVT::v8bf16)
10074 return 0;
10075 if ((Subtarget->isNeonAvailable())) {
10076 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10077 }
10078 return 0;
10079}
10080
10081unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10082 if (RetVT.SimpleTy != MVT::v2f32)
10083 return 0;
10084 if ((Subtarget->isNeonAvailable())) {
10085 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10086 }
10087 return 0;
10088}
10089
10090unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10091 if (RetVT.SimpleTy != MVT::v4f32)
10092 return 0;
10093 if ((Subtarget->isNeonAvailable())) {
10094 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10095 }
10096 return 0;
10097}
10098
10099unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10100 if (RetVT.SimpleTy != MVT::v2f64)
10101 return 0;
10102 if ((Subtarget->isNeonAvailable())) {
10103 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2v2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10104 }
10105 return 0;
10106}
10107
10108unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10109 if (RetVT.SimpleTy != MVT::nxv2i1)
10110 return 0;
10111 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10112 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10113 }
10114 return 0;
10115}
10116
10117unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10118 if (RetVT.SimpleTy != MVT::nxv4i1)
10119 return 0;
10120 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10121 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10122 }
10123 return 0;
10124}
10125
10126unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10127 if (RetVT.SimpleTy != MVT::nxv8i1)
10128 return 0;
10129 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10130 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10131 }
10132 return 0;
10133}
10134
10135unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10136 if (RetVT.SimpleTy != MVT::nxv16i1)
10137 return 0;
10138 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10139 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
10140 }
10141 return 0;
10142}
10143
10144unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10145 if (RetVT.SimpleTy != MVT::nxv16i8)
10146 return 0;
10147 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10148 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10149 }
10150 return 0;
10151}
10152
10153unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10154 if (RetVT.SimpleTy != MVT::nxv8i16)
10155 return 0;
10156 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10157 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10158 }
10159 return 0;
10160}
10161
10162unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10163 if (RetVT.SimpleTy != MVT::nxv4i32)
10164 return 0;
10165 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10166 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10167 }
10168 return 0;
10169}
10170
10171unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10172 if (RetVT.SimpleTy != MVT::nxv2i64)
10173 return 0;
10174 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10175 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10176 }
10177 return 0;
10178}
10179
10180unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10181 if (RetVT.SimpleTy != MVT::nxv2f16)
10182 return 0;
10183 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10184 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10185 }
10186 return 0;
10187}
10188
10189unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10190 if (RetVT.SimpleTy != MVT::nxv4f16)
10191 return 0;
10192 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10193 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10194 }
10195 return 0;
10196}
10197
10198unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10199 if (RetVT.SimpleTy != MVT::nxv8f16)
10200 return 0;
10201 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10202 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10203 }
10204 return 0;
10205}
10206
10207unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10208 if (RetVT.SimpleTy != MVT::nxv8bf16)
10209 return 0;
10210 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10211 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10212 }
10213 return 0;
10214}
10215
10216unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10217 if (RetVT.SimpleTy != MVT::nxv2f32)
10218 return 0;
10219 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10220 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10221 }
10222 return 0;
10223}
10224
10225unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10226 if (RetVT.SimpleTy != MVT::nxv4f32)
10227 return 0;
10228 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10229 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10230 }
10231 return 0;
10232}
10233
10234unsigned fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10235 if (RetVT.SimpleTy != MVT::nxv2f64)
10236 return 0;
10237 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10238 return fastEmitInst_rr(MachineInstOpcode: AArch64::ZIP2_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10239 }
10240 return 0;
10241}
10242
10243unsigned fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10244 switch (VT.SimpleTy) {
10245 case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op1);
10246 case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op1);
10247 case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op1);
10248 case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op1);
10249 case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op1);
10250 case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op1);
10251 case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op1);
10252 case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op1);
10253 case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op1);
10254 case MVT::v4bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v4bf16_rr(RetVT, Op0, Op1);
10255 case MVT::v8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_v8bf16_rr(RetVT, Op0, Op1);
10256 case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op1);
10257 case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op1);
10258 case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op1);
10259 case MVT::nxv2i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i1_rr(RetVT, Op0, Op1);
10260 case MVT::nxv4i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i1_rr(RetVT, Op0, Op1);
10261 case MVT::nxv8i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i1_rr(RetVT, Op0, Op1);
10262 case MVT::nxv16i1: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i1_rr(RetVT, Op0, Op1);
10263 case MVT::nxv16i8: return fastEmit_AArch64ISD_ZIP2_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10264 case MVT::nxv8i16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10265 case MVT::nxv4i32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10266 case MVT::nxv2i64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10267 case MVT::nxv2f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f16_rr(RetVT, Op0, Op1);
10268 case MVT::nxv4f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f16_rr(RetVT, Op0, Op1);
10269 case MVT::nxv8f16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8f16_rr(RetVT, Op0, Op1);
10270 case MVT::nxv8bf16: return fastEmit_AArch64ISD_ZIP2_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
10271 case MVT::nxv2f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f32_rr(RetVT, Op0, Op1);
10272 case MVT::nxv4f32: return fastEmit_AArch64ISD_ZIP2_MVT_nxv4f32_rr(RetVT, Op0, Op1);
10273 case MVT::nxv2f64: return fastEmit_AArch64ISD_ZIP2_MVT_nxv2f64_rr(RetVT, Op0, Op1);
10274 default: return 0;
10275 }
10276}
10277
10278// FastEmit functions for ISD::ABDS.
10279
10280unsigned fastEmit_ISD_ABDS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10281 if (RetVT.SimpleTy != MVT::v8i8)
10282 return 0;
10283 if ((Subtarget->isNeonAvailable())) {
10284 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10285 }
10286 return 0;
10287}
10288
10289unsigned fastEmit_ISD_ABDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10290 if (RetVT.SimpleTy != MVT::v16i8)
10291 return 0;
10292 if ((Subtarget->isNeonAvailable())) {
10293 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10294 }
10295 return 0;
10296}
10297
10298unsigned fastEmit_ISD_ABDS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10299 if (RetVT.SimpleTy != MVT::v4i16)
10300 return 0;
10301 if ((Subtarget->isNeonAvailable())) {
10302 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10303 }
10304 return 0;
10305}
10306
10307unsigned fastEmit_ISD_ABDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10308 if (RetVT.SimpleTy != MVT::v8i16)
10309 return 0;
10310 if ((Subtarget->isNeonAvailable())) {
10311 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10312 }
10313 return 0;
10314}
10315
10316unsigned fastEmit_ISD_ABDS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10317 if (RetVT.SimpleTy != MVT::v2i32)
10318 return 0;
10319 if ((Subtarget->isNeonAvailable())) {
10320 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10321 }
10322 return 0;
10323}
10324
10325unsigned fastEmit_ISD_ABDS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10326 if (RetVT.SimpleTy != MVT::v4i32)
10327 return 0;
10328 if ((Subtarget->isNeonAvailable())) {
10329 return fastEmitInst_rr(MachineInstOpcode: AArch64::SABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10330 }
10331 return 0;
10332}
10333
10334unsigned fastEmit_ISD_ABDS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10335 switch (VT.SimpleTy) {
10336 case MVT::v8i8: return fastEmit_ISD_ABDS_MVT_v8i8_rr(RetVT, Op0, Op1);
10337 case MVT::v16i8: return fastEmit_ISD_ABDS_MVT_v16i8_rr(RetVT, Op0, Op1);
10338 case MVT::v4i16: return fastEmit_ISD_ABDS_MVT_v4i16_rr(RetVT, Op0, Op1);
10339 case MVT::v8i16: return fastEmit_ISD_ABDS_MVT_v8i16_rr(RetVT, Op0, Op1);
10340 case MVT::v2i32: return fastEmit_ISD_ABDS_MVT_v2i32_rr(RetVT, Op0, Op1);
10341 case MVT::v4i32: return fastEmit_ISD_ABDS_MVT_v4i32_rr(RetVT, Op0, Op1);
10342 default: return 0;
10343 }
10344}
10345
10346// FastEmit functions for ISD::ABDU.
10347
10348unsigned fastEmit_ISD_ABDU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10349 if (RetVT.SimpleTy != MVT::v8i8)
10350 return 0;
10351 if ((Subtarget->isNeonAvailable())) {
10352 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10353 }
10354 return 0;
10355}
10356
10357unsigned fastEmit_ISD_ABDU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10358 if (RetVT.SimpleTy != MVT::v16i8)
10359 return 0;
10360 if ((Subtarget->isNeonAvailable())) {
10361 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10362 }
10363 return 0;
10364}
10365
10366unsigned fastEmit_ISD_ABDU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10367 if (RetVT.SimpleTy != MVT::v4i16)
10368 return 0;
10369 if ((Subtarget->isNeonAvailable())) {
10370 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10371 }
10372 return 0;
10373}
10374
10375unsigned fastEmit_ISD_ABDU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10376 if (RetVT.SimpleTy != MVT::v8i16)
10377 return 0;
10378 if ((Subtarget->isNeonAvailable())) {
10379 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10380 }
10381 return 0;
10382}
10383
10384unsigned fastEmit_ISD_ABDU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10385 if (RetVT.SimpleTy != MVT::v2i32)
10386 return 0;
10387 if ((Subtarget->isNeonAvailable())) {
10388 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10389 }
10390 return 0;
10391}
10392
10393unsigned fastEmit_ISD_ABDU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10394 if (RetVT.SimpleTy != MVT::v4i32)
10395 return 0;
10396 if ((Subtarget->isNeonAvailable())) {
10397 return fastEmitInst_rr(MachineInstOpcode: AArch64::UABDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10398 }
10399 return 0;
10400}
10401
10402unsigned fastEmit_ISD_ABDU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10403 switch (VT.SimpleTy) {
10404 case MVT::v8i8: return fastEmit_ISD_ABDU_MVT_v8i8_rr(RetVT, Op0, Op1);
10405 case MVT::v16i8: return fastEmit_ISD_ABDU_MVT_v16i8_rr(RetVT, Op0, Op1);
10406 case MVT::v4i16: return fastEmit_ISD_ABDU_MVT_v4i16_rr(RetVT, Op0, Op1);
10407 case MVT::v8i16: return fastEmit_ISD_ABDU_MVT_v8i16_rr(RetVT, Op0, Op1);
10408 case MVT::v2i32: return fastEmit_ISD_ABDU_MVT_v2i32_rr(RetVT, Op0, Op1);
10409 case MVT::v4i32: return fastEmit_ISD_ABDU_MVT_v4i32_rr(RetVT, Op0, Op1);
10410 default: return 0;
10411 }
10412}
10413
10414// FastEmit functions for ISD::ADD.
10415
10416unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10417 if (RetVT.SimpleTy != MVT::i32)
10418 return 0;
10419 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10420}
10421
10422unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10423 if (RetVT.SimpleTy != MVT::i64)
10424 return 0;
10425 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10426}
10427
10428unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10429 if (RetVT.SimpleTy != MVT::v8i8)
10430 return 0;
10431 if ((Subtarget->isNeonAvailable())) {
10432 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10433 }
10434 return 0;
10435}
10436
10437unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10438 if (RetVT.SimpleTy != MVT::v16i8)
10439 return 0;
10440 if ((Subtarget->isNeonAvailable())) {
10441 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10442 }
10443 return 0;
10444}
10445
10446unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10447 if (RetVT.SimpleTy != MVT::v4i16)
10448 return 0;
10449 if ((Subtarget->isNeonAvailable())) {
10450 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10451 }
10452 return 0;
10453}
10454
10455unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10456 if (RetVT.SimpleTy != MVT::v8i16)
10457 return 0;
10458 if ((Subtarget->isNeonAvailable())) {
10459 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10460 }
10461 return 0;
10462}
10463
10464unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10465 if (RetVT.SimpleTy != MVT::v2i32)
10466 return 0;
10467 if ((Subtarget->isNeonAvailable())) {
10468 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10469 }
10470 return 0;
10471}
10472
10473unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10474 if (RetVT.SimpleTy != MVT::v4i32)
10475 return 0;
10476 if ((Subtarget->isNeonAvailable())) {
10477 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10478 }
10479 return 0;
10480}
10481
10482unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10483 if (RetVT.SimpleTy != MVT::v1i64)
10484 return 0;
10485 if ((Subtarget->isNeonAvailable())) {
10486 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
10487 }
10488 return 0;
10489}
10490
10491unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10492 if (RetVT.SimpleTy != MVT::v2i64)
10493 return 0;
10494 if ((Subtarget->isNeonAvailable())) {
10495 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
10496 }
10497 return 0;
10498}
10499
10500unsigned fastEmit_ISD_ADD_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10501 if (RetVT.SimpleTy != MVT::nxv16i8)
10502 return 0;
10503 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10504 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
10505 }
10506 return 0;
10507}
10508
10509unsigned fastEmit_ISD_ADD_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10510 if (RetVT.SimpleTy != MVT::nxv8i16)
10511 return 0;
10512 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10513 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
10514 }
10515 return 0;
10516}
10517
10518unsigned fastEmit_ISD_ADD_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10519 if (RetVT.SimpleTy != MVT::nxv4i32)
10520 return 0;
10521 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10522 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
10523 }
10524 return 0;
10525}
10526
10527unsigned fastEmit_ISD_ADD_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10528 if (RetVT.SimpleTy != MVT::nxv2i64)
10529 return 0;
10530 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10531 return fastEmitInst_rr(MachineInstOpcode: AArch64::ADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
10532 }
10533 return 0;
10534}
10535
10536unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10537 switch (VT.SimpleTy) {
10538 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op1);
10539 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op1);
10540 case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op1);
10541 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op1);
10542 case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op1);
10543 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op1);
10544 case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op1);
10545 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op1);
10546 case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op1);
10547 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op1);
10548 case MVT::nxv16i8: return fastEmit_ISD_ADD_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10549 case MVT::nxv8i16: return fastEmit_ISD_ADD_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10550 case MVT::nxv4i32: return fastEmit_ISD_ADD_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10551 case MVT::nxv2i64: return fastEmit_ISD_ADD_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10552 default: return 0;
10553 }
10554}
10555
10556// FastEmit functions for ISD::AND.
10557
10558unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10559 if (RetVT.SimpleTy != MVT::i32)
10560 return 0;
10561 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
10562}
10563
10564unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10565 if (RetVT.SimpleTy != MVT::i64)
10566 return 0;
10567 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
10568}
10569
10570unsigned fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10571 if (RetVT.SimpleTy != MVT::v8i8)
10572 return 0;
10573 if ((Subtarget->isNeonAvailable())) {
10574 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10575 }
10576 return 0;
10577}
10578
10579unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10580 if (RetVT.SimpleTy != MVT::v16i8)
10581 return 0;
10582 if ((Subtarget->isNeonAvailable())) {
10583 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10584 }
10585 return 0;
10586}
10587
10588unsigned fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10589 if (RetVT.SimpleTy != MVT::v4i16)
10590 return 0;
10591 if ((Subtarget->isNeonAvailable())) {
10592 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10593 }
10594 return 0;
10595}
10596
10597unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10598 if (RetVT.SimpleTy != MVT::v8i16)
10599 return 0;
10600 if ((Subtarget->isNeonAvailable())) {
10601 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10602 }
10603 return 0;
10604}
10605
10606unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10607 if (RetVT.SimpleTy != MVT::v2i32)
10608 return 0;
10609 if ((Subtarget->isNeonAvailable())) {
10610 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10611 }
10612 return 0;
10613}
10614
10615unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10616 if (RetVT.SimpleTy != MVT::v4i32)
10617 return 0;
10618 if ((Subtarget->isNeonAvailable())) {
10619 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10620 }
10621 return 0;
10622}
10623
10624unsigned fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10625 if (RetVT.SimpleTy != MVT::v1i64)
10626 return 0;
10627 if ((Subtarget->isNeonAvailable())) {
10628 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10629 }
10630 return 0;
10631}
10632
10633unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10634 if (RetVT.SimpleTy != MVT::v2i64)
10635 return 0;
10636 if ((Subtarget->isNeonAvailable())) {
10637 return fastEmitInst_rr(MachineInstOpcode: AArch64::ANDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10638 }
10639 return 0;
10640}
10641
10642unsigned fastEmit_ISD_AND_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10643 if (RetVT.SimpleTy != MVT::nxv16i8)
10644 return 0;
10645 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10646 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10647 }
10648 return 0;
10649}
10650
10651unsigned fastEmit_ISD_AND_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10652 if (RetVT.SimpleTy != MVT::nxv8i16)
10653 return 0;
10654 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10655 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10656 }
10657 return 0;
10658}
10659
10660unsigned fastEmit_ISD_AND_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10661 if (RetVT.SimpleTy != MVT::nxv4i32)
10662 return 0;
10663 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10664 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10665 }
10666 return 0;
10667}
10668
10669unsigned fastEmit_ISD_AND_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10670 if (RetVT.SimpleTy != MVT::nxv2i64)
10671 return 0;
10672 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10673 return fastEmitInst_rr(MachineInstOpcode: AArch64::AND_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
10674 }
10675 return 0;
10676}
10677
10678unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10679 switch (VT.SimpleTy) {
10680 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op1);
10681 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op1);
10682 case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op1);
10683 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op1);
10684 case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op1);
10685 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op1);
10686 case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op1);
10687 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op1);
10688 case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op1);
10689 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op1);
10690 case MVT::nxv16i8: return fastEmit_ISD_AND_MVT_nxv16i8_rr(RetVT, Op0, Op1);
10691 case MVT::nxv8i16: return fastEmit_ISD_AND_MVT_nxv8i16_rr(RetVT, Op0, Op1);
10692 case MVT::nxv4i32: return fastEmit_ISD_AND_MVT_nxv4i32_rr(RetVT, Op0, Op1);
10693 case MVT::nxv2i64: return fastEmit_ISD_AND_MVT_nxv2i64_rr(RetVT, Op0, Op1);
10694 default: return 0;
10695 }
10696}
10697
10698// FastEmit functions for ISD::AVGCEILS.
10699
10700unsigned fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10701 if (RetVT.SimpleTy != MVT::v8i8)
10702 return 0;
10703 if ((Subtarget->isNeonAvailable())) {
10704 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10705 }
10706 return 0;
10707}
10708
10709unsigned fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10710 if (RetVT.SimpleTy != MVT::v16i8)
10711 return 0;
10712 if ((Subtarget->isNeonAvailable())) {
10713 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10714 }
10715 return 0;
10716}
10717
10718unsigned fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10719 if (RetVT.SimpleTy != MVT::v4i16)
10720 return 0;
10721 if ((Subtarget->isNeonAvailable())) {
10722 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10723 }
10724 return 0;
10725}
10726
10727unsigned fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10728 if (RetVT.SimpleTy != MVT::v8i16)
10729 return 0;
10730 if ((Subtarget->isNeonAvailable())) {
10731 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10732 }
10733 return 0;
10734}
10735
10736unsigned fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10737 if (RetVT.SimpleTy != MVT::v2i32)
10738 return 0;
10739 if ((Subtarget->isNeonAvailable())) {
10740 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10741 }
10742 return 0;
10743}
10744
10745unsigned fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10746 if (RetVT.SimpleTy != MVT::v4i32)
10747 return 0;
10748 if ((Subtarget->isNeonAvailable())) {
10749 return fastEmitInst_rr(MachineInstOpcode: AArch64::SRHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10750 }
10751 return 0;
10752}
10753
10754unsigned fastEmit_ISD_AVGCEILS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10755 switch (VT.SimpleTy) {
10756 case MVT::v8i8: return fastEmit_ISD_AVGCEILS_MVT_v8i8_rr(RetVT, Op0, Op1);
10757 case MVT::v16i8: return fastEmit_ISD_AVGCEILS_MVT_v16i8_rr(RetVT, Op0, Op1);
10758 case MVT::v4i16: return fastEmit_ISD_AVGCEILS_MVT_v4i16_rr(RetVT, Op0, Op1);
10759 case MVT::v8i16: return fastEmit_ISD_AVGCEILS_MVT_v8i16_rr(RetVT, Op0, Op1);
10760 case MVT::v2i32: return fastEmit_ISD_AVGCEILS_MVT_v2i32_rr(RetVT, Op0, Op1);
10761 case MVT::v4i32: return fastEmit_ISD_AVGCEILS_MVT_v4i32_rr(RetVT, Op0, Op1);
10762 default: return 0;
10763 }
10764}
10765
10766// FastEmit functions for ISD::AVGCEILU.
10767
10768unsigned fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10769 if (RetVT.SimpleTy != MVT::v8i8)
10770 return 0;
10771 if ((Subtarget->isNeonAvailable())) {
10772 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10773 }
10774 return 0;
10775}
10776
10777unsigned fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10778 if (RetVT.SimpleTy != MVT::v16i8)
10779 return 0;
10780 if ((Subtarget->isNeonAvailable())) {
10781 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10782 }
10783 return 0;
10784}
10785
10786unsigned fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10787 if (RetVT.SimpleTy != MVT::v4i16)
10788 return 0;
10789 if ((Subtarget->isNeonAvailable())) {
10790 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10791 }
10792 return 0;
10793}
10794
10795unsigned fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10796 if (RetVT.SimpleTy != MVT::v8i16)
10797 return 0;
10798 if ((Subtarget->isNeonAvailable())) {
10799 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10800 }
10801 return 0;
10802}
10803
10804unsigned fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10805 if (RetVT.SimpleTy != MVT::v2i32)
10806 return 0;
10807 if ((Subtarget->isNeonAvailable())) {
10808 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10809 }
10810 return 0;
10811}
10812
10813unsigned fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10814 if (RetVT.SimpleTy != MVT::v4i32)
10815 return 0;
10816 if ((Subtarget->isNeonAvailable())) {
10817 return fastEmitInst_rr(MachineInstOpcode: AArch64::URHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10818 }
10819 return 0;
10820}
10821
10822unsigned fastEmit_ISD_AVGCEILU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10823 switch (VT.SimpleTy) {
10824 case MVT::v8i8: return fastEmit_ISD_AVGCEILU_MVT_v8i8_rr(RetVT, Op0, Op1);
10825 case MVT::v16i8: return fastEmit_ISD_AVGCEILU_MVT_v16i8_rr(RetVT, Op0, Op1);
10826 case MVT::v4i16: return fastEmit_ISD_AVGCEILU_MVT_v4i16_rr(RetVT, Op0, Op1);
10827 case MVT::v8i16: return fastEmit_ISD_AVGCEILU_MVT_v8i16_rr(RetVT, Op0, Op1);
10828 case MVT::v2i32: return fastEmit_ISD_AVGCEILU_MVT_v2i32_rr(RetVT, Op0, Op1);
10829 case MVT::v4i32: return fastEmit_ISD_AVGCEILU_MVT_v4i32_rr(RetVT, Op0, Op1);
10830 default: return 0;
10831 }
10832}
10833
10834// FastEmit functions for ISD::AVGFLOORS.
10835
10836unsigned fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10837 if (RetVT.SimpleTy != MVT::v8i8)
10838 return 0;
10839 if ((Subtarget->isNeonAvailable())) {
10840 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10841 }
10842 return 0;
10843}
10844
10845unsigned fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10846 if (RetVT.SimpleTy != MVT::v16i8)
10847 return 0;
10848 if ((Subtarget->isNeonAvailable())) {
10849 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10850 }
10851 return 0;
10852}
10853
10854unsigned fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10855 if (RetVT.SimpleTy != MVT::v4i16)
10856 return 0;
10857 if ((Subtarget->isNeonAvailable())) {
10858 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10859 }
10860 return 0;
10861}
10862
10863unsigned fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10864 if (RetVT.SimpleTy != MVT::v8i16)
10865 return 0;
10866 if ((Subtarget->isNeonAvailable())) {
10867 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10868 }
10869 return 0;
10870}
10871
10872unsigned fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10873 if (RetVT.SimpleTy != MVT::v2i32)
10874 return 0;
10875 if ((Subtarget->isNeonAvailable())) {
10876 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10877 }
10878 return 0;
10879}
10880
10881unsigned fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10882 if (RetVT.SimpleTy != MVT::v4i32)
10883 return 0;
10884 if ((Subtarget->isNeonAvailable())) {
10885 return fastEmitInst_rr(MachineInstOpcode: AArch64::SHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10886 }
10887 return 0;
10888}
10889
10890unsigned fastEmit_ISD_AVGFLOORS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10891 switch (VT.SimpleTy) {
10892 case MVT::v8i8: return fastEmit_ISD_AVGFLOORS_MVT_v8i8_rr(RetVT, Op0, Op1);
10893 case MVT::v16i8: return fastEmit_ISD_AVGFLOORS_MVT_v16i8_rr(RetVT, Op0, Op1);
10894 case MVT::v4i16: return fastEmit_ISD_AVGFLOORS_MVT_v4i16_rr(RetVT, Op0, Op1);
10895 case MVT::v8i16: return fastEmit_ISD_AVGFLOORS_MVT_v8i16_rr(RetVT, Op0, Op1);
10896 case MVT::v2i32: return fastEmit_ISD_AVGFLOORS_MVT_v2i32_rr(RetVT, Op0, Op1);
10897 case MVT::v4i32: return fastEmit_ISD_AVGFLOORS_MVT_v4i32_rr(RetVT, Op0, Op1);
10898 default: return 0;
10899 }
10900}
10901
10902// FastEmit functions for ISD::AVGFLOORU.
10903
10904unsigned fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10905 if (RetVT.SimpleTy != MVT::v8i8)
10906 return 0;
10907 if ((Subtarget->isNeonAvailable())) {
10908 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
10909 }
10910 return 0;
10911}
10912
10913unsigned fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10914 if (RetVT.SimpleTy != MVT::v16i8)
10915 return 0;
10916 if ((Subtarget->isNeonAvailable())) {
10917 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
10918 }
10919 return 0;
10920}
10921
10922unsigned fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10923 if (RetVT.SimpleTy != MVT::v4i16)
10924 return 0;
10925 if ((Subtarget->isNeonAvailable())) {
10926 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
10927 }
10928 return 0;
10929}
10930
10931unsigned fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10932 if (RetVT.SimpleTy != MVT::v8i16)
10933 return 0;
10934 if ((Subtarget->isNeonAvailable())) {
10935 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
10936 }
10937 return 0;
10938}
10939
10940unsigned fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10941 if (RetVT.SimpleTy != MVT::v2i32)
10942 return 0;
10943 if ((Subtarget->isNeonAvailable())) {
10944 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
10945 }
10946 return 0;
10947}
10948
10949unsigned fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10950 if (RetVT.SimpleTy != MVT::v4i32)
10951 return 0;
10952 if ((Subtarget->isNeonAvailable())) {
10953 return fastEmitInst_rr(MachineInstOpcode: AArch64::UHADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
10954 }
10955 return 0;
10956}
10957
10958unsigned fastEmit_ISD_AVGFLOORU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
10959 switch (VT.SimpleTy) {
10960 case MVT::v8i8: return fastEmit_ISD_AVGFLOORU_MVT_v8i8_rr(RetVT, Op0, Op1);
10961 case MVT::v16i8: return fastEmit_ISD_AVGFLOORU_MVT_v16i8_rr(RetVT, Op0, Op1);
10962 case MVT::v4i16: return fastEmit_ISD_AVGFLOORU_MVT_v4i16_rr(RetVT, Op0, Op1);
10963 case MVT::v8i16: return fastEmit_ISD_AVGFLOORU_MVT_v8i16_rr(RetVT, Op0, Op1);
10964 case MVT::v2i32: return fastEmit_ISD_AVGFLOORU_MVT_v2i32_rr(RetVT, Op0, Op1);
10965 case MVT::v4i32: return fastEmit_ISD_AVGFLOORU_MVT_v4i32_rr(RetVT, Op0, Op1);
10966 default: return 0;
10967 }
10968}
10969
10970// FastEmit functions for ISD::CONCAT_VECTORS.
10971
10972unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10973 if (RetVT.SimpleTy != MVT::nxv2i1)
10974 return 0;
10975 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10976 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_D, RC: &AArch64::PPRRegClass, Op0, Op1);
10977 }
10978 return 0;
10979}
10980
10981unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10982 if (RetVT.SimpleTy != MVT::nxv4i1)
10983 return 0;
10984 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10985 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_S, RC: &AArch64::PPRRegClass, Op0, Op1);
10986 }
10987 return 0;
10988}
10989
10990unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
10991 if (RetVT.SimpleTy != MVT::nxv8i1)
10992 return 0;
10993 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
10994 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_H, RC: &AArch64::PPRRegClass, Op0, Op1);
10995 }
10996 return 0;
10997}
10998
10999unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11000 if (RetVT.SimpleTy != MVT::nxv16i1)
11001 return 0;
11002 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11003 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_PPP_B, RC: &AArch64::PPRRegClass, Op0, Op1);
11004 }
11005 return 0;
11006}
11007
11008unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11009 if (RetVT.SimpleTy != MVT::nxv4f16)
11010 return 0;
11011 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11012 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11013 }
11014 return 0;
11015}
11016
11017unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11018 if (RetVT.SimpleTy != MVT::nxv8f16)
11019 return 0;
11020 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11021 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11022 }
11023 return 0;
11024}
11025
11026unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11027 if (RetVT.SimpleTy != MVT::nxv4bf16)
11028 return 0;
11029 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11030 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11031 }
11032 return 0;
11033}
11034
11035unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11036 if (RetVT.SimpleTy != MVT::nxv8bf16)
11037 return 0;
11038 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11039 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11040 }
11041 return 0;
11042}
11043
11044unsigned fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11045 if (RetVT.SimpleTy != MVT::nxv4f32)
11046 return 0;
11047 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11048 return fastEmitInst_rr(MachineInstOpcode: AArch64::UZP1_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11049 }
11050 return 0;
11051}
11052
11053unsigned fastEmit_ISD_CONCAT_VECTORS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11054 switch (VT.SimpleTy) {
11055 case MVT::nxv1i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv1i1_rr(RetVT, Op0, Op1);
11056 case MVT::nxv2i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2i1_rr(RetVT, Op0, Op1);
11057 case MVT::nxv4i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4i1_rr(RetVT, Op0, Op1);
11058 case MVT::nxv8i1: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv8i1_rr(RetVT, Op0, Op1);
11059 case MVT::nxv2f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f16_rr(RetVT, Op0, Op1);
11060 case MVT::nxv4f16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4f16_rr(RetVT, Op0, Op1);
11061 case MVT::nxv2bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2bf16_rr(RetVT, Op0, Op1);
11062 case MVT::nxv4bf16: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv4bf16_rr(RetVT, Op0, Op1);
11063 case MVT::nxv2f32: return fastEmit_ISD_CONCAT_VECTORS_MVT_nxv2f32_rr(RetVT, Op0, Op1);
11064 default: return 0;
11065 }
11066}
11067
11068// FastEmit functions for ISD::FADD.
11069
11070unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11071 if (RetVT.SimpleTy != MVT::f16)
11072 return 0;
11073 if ((Subtarget->hasFullFP16())) {
11074 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11075 }
11076 return 0;
11077}
11078
11079unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11080 if (RetVT.SimpleTy != MVT::f32)
11081 return 0;
11082 if ((Subtarget->hasFPARMv8())) {
11083 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11084 }
11085 return 0;
11086}
11087
11088unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11089 if (RetVT.SimpleTy != MVT::f64)
11090 return 0;
11091 if ((Subtarget->hasFPARMv8())) {
11092 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11093 }
11094 return 0;
11095}
11096
11097unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11098 if (RetVT.SimpleTy != MVT::v4f16)
11099 return 0;
11100 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11101 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11102 }
11103 return 0;
11104}
11105
11106unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11107 if (RetVT.SimpleTy != MVT::v8f16)
11108 return 0;
11109 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11110 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11111 }
11112 return 0;
11113}
11114
11115unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11116 if (RetVT.SimpleTy != MVT::v2f32)
11117 return 0;
11118 if ((Subtarget->isNeonAvailable())) {
11119 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11120 }
11121 return 0;
11122}
11123
11124unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11125 if (RetVT.SimpleTy != MVT::v4f32)
11126 return 0;
11127 if ((Subtarget->isNeonAvailable())) {
11128 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11129 }
11130 return 0;
11131}
11132
11133unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11134 if (RetVT.SimpleTy != MVT::v2f64)
11135 return 0;
11136 if ((Subtarget->isNeonAvailable())) {
11137 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11138 }
11139 return 0;
11140}
11141
11142unsigned fastEmit_ISD_FADD_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11143 if (RetVT.SimpleTy != MVT::nxv8f16)
11144 return 0;
11145 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11146 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11147 }
11148 return 0;
11149}
11150
11151unsigned fastEmit_ISD_FADD_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11152 if (RetVT.SimpleTy != MVT::nxv8bf16)
11153 return 0;
11154 if ((Subtarget->hasB16B16()) && (Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2()))) {
11155 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFADD_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11156 }
11157 return 0;
11158}
11159
11160unsigned fastEmit_ISD_FADD_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11161 if (RetVT.SimpleTy != MVT::nxv4f32)
11162 return 0;
11163 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11164 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11165 }
11166 return 0;
11167}
11168
11169unsigned fastEmit_ISD_FADD_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11170 if (RetVT.SimpleTy != MVT::nxv2f64)
11171 return 0;
11172 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11173 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11174 }
11175 return 0;
11176}
11177
11178unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11179 switch (VT.SimpleTy) {
11180 case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op1);
11181 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op1);
11182 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op1);
11183 case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
11184 case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
11185 case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
11186 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
11187 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
11188 case MVT::nxv8f16: return fastEmit_ISD_FADD_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11189 case MVT::nxv8bf16: return fastEmit_ISD_FADD_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11190 case MVT::nxv4f32: return fastEmit_ISD_FADD_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11191 case MVT::nxv2f64: return fastEmit_ISD_FADD_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11192 default: return 0;
11193 }
11194}
11195
11196// FastEmit functions for ISD::FDIV.
11197
11198unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11199 if (RetVT.SimpleTy != MVT::f16)
11200 return 0;
11201 if ((Subtarget->hasFullFP16())) {
11202 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11203 }
11204 return 0;
11205}
11206
11207unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11208 if (RetVT.SimpleTy != MVT::f32)
11209 return 0;
11210 if ((Subtarget->hasFPARMv8())) {
11211 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11212 }
11213 return 0;
11214}
11215
11216unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11217 if (RetVT.SimpleTy != MVT::f64)
11218 return 0;
11219 if ((Subtarget->hasFPARMv8())) {
11220 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11221 }
11222 return 0;
11223}
11224
11225unsigned fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11226 if (RetVT.SimpleTy != MVT::v4f16)
11227 return 0;
11228 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11229 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11230 }
11231 return 0;
11232}
11233
11234unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11235 if (RetVT.SimpleTy != MVT::v8f16)
11236 return 0;
11237 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11238 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11239 }
11240 return 0;
11241}
11242
11243unsigned fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11244 if (RetVT.SimpleTy != MVT::v2f32)
11245 return 0;
11246 if ((Subtarget->isNeonAvailable())) {
11247 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11248 }
11249 return 0;
11250}
11251
11252unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11253 if (RetVT.SimpleTy != MVT::v4f32)
11254 return 0;
11255 if ((Subtarget->isNeonAvailable())) {
11256 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11257 }
11258 return 0;
11259}
11260
11261unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11262 if (RetVT.SimpleTy != MVT::v2f64)
11263 return 0;
11264 if ((Subtarget->isNeonAvailable())) {
11265 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11266 }
11267 return 0;
11268}
11269
11270unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11271 switch (VT.SimpleTy) {
11272 case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
11273 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
11274 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
11275 case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
11276 case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
11277 case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
11278 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
11279 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
11280 default: return 0;
11281 }
11282}
11283
11284// FastEmit functions for ISD::FMAXIMUM.
11285
11286unsigned fastEmit_ISD_FMAXIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11287 if (RetVT.SimpleTy != MVT::f16)
11288 return 0;
11289 if ((Subtarget->hasFullFP16())) {
11290 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11291 }
11292 return 0;
11293}
11294
11295unsigned fastEmit_ISD_FMAXIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11296 if (RetVT.SimpleTy != MVT::f32)
11297 return 0;
11298 if ((Subtarget->hasFPARMv8())) {
11299 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11300 }
11301 return 0;
11302}
11303
11304unsigned fastEmit_ISD_FMAXIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11305 if (RetVT.SimpleTy != MVT::f64)
11306 return 0;
11307 if ((Subtarget->hasFPARMv8())) {
11308 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11309 }
11310 return 0;
11311}
11312
11313unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11314 if (RetVT.SimpleTy != MVT::v4f16)
11315 return 0;
11316 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11317 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11318 }
11319 return 0;
11320}
11321
11322unsigned fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11323 if (RetVT.SimpleTy != MVT::v8f16)
11324 return 0;
11325 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11326 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11327 }
11328 return 0;
11329}
11330
11331unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11332 if (RetVT.SimpleTy != MVT::v2f32)
11333 return 0;
11334 if ((Subtarget->isNeonAvailable())) {
11335 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11336 }
11337 return 0;
11338}
11339
11340unsigned fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11341 if (RetVT.SimpleTy != MVT::v4f32)
11342 return 0;
11343 if ((Subtarget->isNeonAvailable())) {
11344 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11345 }
11346 return 0;
11347}
11348
11349unsigned fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11350 if (RetVT.SimpleTy != MVT::v1f64)
11351 return 0;
11352 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11353}
11354
11355unsigned fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11356 if (RetVT.SimpleTy != MVT::v2f64)
11357 return 0;
11358 if ((Subtarget->isNeonAvailable())) {
11359 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11360 }
11361 return 0;
11362}
11363
11364unsigned fastEmit_ISD_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11365 switch (VT.SimpleTy) {
11366 case MVT::f16: return fastEmit_ISD_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11367 case MVT::f32: return fastEmit_ISD_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11368 case MVT::f64: return fastEmit_ISD_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11369 case MVT::v4f16: return fastEmit_ISD_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11370 case MVT::v8f16: return fastEmit_ISD_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11371 case MVT::v2f32: return fastEmit_ISD_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11372 case MVT::v4f32: return fastEmit_ISD_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11373 case MVT::v1f64: return fastEmit_ISD_FMAXIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11374 case MVT::v2f64: return fastEmit_ISD_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11375 default: return 0;
11376 }
11377}
11378
11379// FastEmit functions for ISD::FMAXNUM.
11380
11381unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11382 if (RetVT.SimpleTy != MVT::f16)
11383 return 0;
11384 if ((Subtarget->hasFullFP16())) {
11385 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11386 }
11387 return 0;
11388}
11389
11390unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11391 if (RetVT.SimpleTy != MVT::f32)
11392 return 0;
11393 if ((Subtarget->hasFPARMv8())) {
11394 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11395 }
11396 return 0;
11397}
11398
11399unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11400 if (RetVT.SimpleTy != MVT::f64)
11401 return 0;
11402 if ((Subtarget->hasFPARMv8())) {
11403 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11404 }
11405 return 0;
11406}
11407
11408unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11409 if (RetVT.SimpleTy != MVT::v4f16)
11410 return 0;
11411 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11412 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11413 }
11414 return 0;
11415}
11416
11417unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11418 if (RetVT.SimpleTy != MVT::v8f16)
11419 return 0;
11420 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11421 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11422 }
11423 return 0;
11424}
11425
11426unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11427 if (RetVT.SimpleTy != MVT::v2f32)
11428 return 0;
11429 if ((Subtarget->isNeonAvailable())) {
11430 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11431 }
11432 return 0;
11433}
11434
11435unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11436 if (RetVT.SimpleTy != MVT::v4f32)
11437 return 0;
11438 if ((Subtarget->isNeonAvailable())) {
11439 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11440 }
11441 return 0;
11442}
11443
11444unsigned fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11445 if (RetVT.SimpleTy != MVT::v1f64)
11446 return 0;
11447 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11448}
11449
11450unsigned fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11451 if (RetVT.SimpleTy != MVT::v2f64)
11452 return 0;
11453 if ((Subtarget->isNeonAvailable())) {
11454 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11455 }
11456 return 0;
11457}
11458
11459unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11460 switch (VT.SimpleTy) {
11461 case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
11462 case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
11463 case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
11464 case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11465 case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11466 case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11467 case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11468 case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11469 case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11470 default: return 0;
11471 }
11472}
11473
11474// FastEmit functions for ISD::FMINIMUM.
11475
11476unsigned fastEmit_ISD_FMINIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11477 if (RetVT.SimpleTy != MVT::f16)
11478 return 0;
11479 if ((Subtarget->hasFullFP16())) {
11480 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11481 }
11482 return 0;
11483}
11484
11485unsigned fastEmit_ISD_FMINIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11486 if (RetVT.SimpleTy != MVT::f32)
11487 return 0;
11488 if ((Subtarget->hasFPARMv8())) {
11489 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11490 }
11491 return 0;
11492}
11493
11494unsigned fastEmit_ISD_FMINIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11495 if (RetVT.SimpleTy != MVT::f64)
11496 return 0;
11497 if ((Subtarget->hasFPARMv8())) {
11498 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11499 }
11500 return 0;
11501}
11502
11503unsigned fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11504 if (RetVT.SimpleTy != MVT::v4f16)
11505 return 0;
11506 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11507 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11508 }
11509 return 0;
11510}
11511
11512unsigned fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11513 if (RetVT.SimpleTy != MVT::v8f16)
11514 return 0;
11515 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11516 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11517 }
11518 return 0;
11519}
11520
11521unsigned fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11522 if (RetVT.SimpleTy != MVT::v2f32)
11523 return 0;
11524 if ((Subtarget->isNeonAvailable())) {
11525 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11526 }
11527 return 0;
11528}
11529
11530unsigned fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11531 if (RetVT.SimpleTy != MVT::v4f32)
11532 return 0;
11533 if ((Subtarget->isNeonAvailable())) {
11534 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11535 }
11536 return 0;
11537}
11538
11539unsigned fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11540 if (RetVT.SimpleTy != MVT::v1f64)
11541 return 0;
11542 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11543}
11544
11545unsigned fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11546 if (RetVT.SimpleTy != MVT::v2f64)
11547 return 0;
11548 if ((Subtarget->isNeonAvailable())) {
11549 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11550 }
11551 return 0;
11552}
11553
11554unsigned fastEmit_ISD_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11555 switch (VT.SimpleTy) {
11556 case MVT::f16: return fastEmit_ISD_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
11557 case MVT::f32: return fastEmit_ISD_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
11558 case MVT::f64: return fastEmit_ISD_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
11559 case MVT::v4f16: return fastEmit_ISD_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11560 case MVT::v8f16: return fastEmit_ISD_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11561 case MVT::v2f32: return fastEmit_ISD_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11562 case MVT::v4f32: return fastEmit_ISD_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11563 case MVT::v1f64: return fastEmit_ISD_FMINIMUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11564 case MVT::v2f64: return fastEmit_ISD_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11565 default: return 0;
11566 }
11567}
11568
11569// FastEmit functions for ISD::FMINNUM.
11570
11571unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11572 if (RetVT.SimpleTy != MVT::f16)
11573 return 0;
11574 if ((Subtarget->hasFullFP16())) {
11575 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11576 }
11577 return 0;
11578}
11579
11580unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11581 if (RetVT.SimpleTy != MVT::f32)
11582 return 0;
11583 if ((Subtarget->hasFPARMv8())) {
11584 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11585 }
11586 return 0;
11587}
11588
11589unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11590 if (RetVT.SimpleTy != MVT::f64)
11591 return 0;
11592 if ((Subtarget->hasFPARMv8())) {
11593 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11594 }
11595 return 0;
11596}
11597
11598unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11599 if (RetVT.SimpleTy != MVT::v4f16)
11600 return 0;
11601 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11602 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11603 }
11604 return 0;
11605}
11606
11607unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11608 if (RetVT.SimpleTy != MVT::v8f16)
11609 return 0;
11610 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11611 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11612 }
11613 return 0;
11614}
11615
11616unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11617 if (RetVT.SimpleTy != MVT::v2f32)
11618 return 0;
11619 if ((Subtarget->isNeonAvailable())) {
11620 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11621 }
11622 return 0;
11623}
11624
11625unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11626 if (RetVT.SimpleTy != MVT::v4f32)
11627 return 0;
11628 if ((Subtarget->isNeonAvailable())) {
11629 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11630 }
11631 return 0;
11632}
11633
11634unsigned fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11635 if (RetVT.SimpleTy != MVT::v1f64)
11636 return 0;
11637 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11638}
11639
11640unsigned fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11641 if (RetVT.SimpleTy != MVT::v2f64)
11642 return 0;
11643 if ((Subtarget->isNeonAvailable())) {
11644 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11645 }
11646 return 0;
11647}
11648
11649unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11650 switch (VT.SimpleTy) {
11651 case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
11652 case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
11653 case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
11654 case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
11655 case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
11656 case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
11657 case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
11658 case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op1);
11659 case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
11660 default: return 0;
11661 }
11662}
11663
11664// FastEmit functions for ISD::FMUL.
11665
11666unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11667 if (RetVT.SimpleTy != MVT::f16)
11668 return 0;
11669 if ((Subtarget->hasFullFP16())) {
11670 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11671 }
11672 return 0;
11673}
11674
11675unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11676 if (RetVT.SimpleTy != MVT::f32)
11677 return 0;
11678 if ((Subtarget->hasFPARMv8())) {
11679 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11680 }
11681 return 0;
11682}
11683
11684unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11685 if (RetVT.SimpleTy != MVT::f64)
11686 return 0;
11687 if ((Subtarget->hasFPARMv8())) {
11688 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11689 }
11690 return 0;
11691}
11692
11693unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11694 if (RetVT.SimpleTy != MVT::v4f16)
11695 return 0;
11696 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11697 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11698 }
11699 return 0;
11700}
11701
11702unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11703 if (RetVT.SimpleTy != MVT::v8f16)
11704 return 0;
11705 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11706 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11707 }
11708 return 0;
11709}
11710
11711unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11712 if (RetVT.SimpleTy != MVT::v2f32)
11713 return 0;
11714 if ((Subtarget->isNeonAvailable())) {
11715 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11716 }
11717 return 0;
11718}
11719
11720unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11721 if (RetVT.SimpleTy != MVT::v4f32)
11722 return 0;
11723 if ((Subtarget->isNeonAvailable())) {
11724 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11725 }
11726 return 0;
11727}
11728
11729unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11730 if (RetVT.SimpleTy != MVT::v2f64)
11731 return 0;
11732 if ((Subtarget->isNeonAvailable())) {
11733 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11734 }
11735 return 0;
11736}
11737
11738unsigned fastEmit_ISD_FMUL_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11739 if (RetVT.SimpleTy != MVT::nxv8f16)
11740 return 0;
11741 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11742 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11743 }
11744 return 0;
11745}
11746
11747unsigned fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11748 if (RetVT.SimpleTy != MVT::nxv8bf16)
11749 return 0;
11750 if ((Subtarget->hasB16B16()) && (Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2()))) {
11751 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFMUL_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11752 }
11753 return 0;
11754}
11755
11756unsigned fastEmit_ISD_FMUL_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11757 if (RetVT.SimpleTy != MVT::nxv4f32)
11758 return 0;
11759 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11760 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11761 }
11762 return 0;
11763}
11764
11765unsigned fastEmit_ISD_FMUL_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11766 if (RetVT.SimpleTy != MVT::nxv2f64)
11767 return 0;
11768 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11769 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMUL_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11770 }
11771 return 0;
11772}
11773
11774unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11775 switch (VT.SimpleTy) {
11776 case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
11777 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
11778 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
11779 case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
11780 case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
11781 case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
11782 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
11783 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
11784 case MVT::nxv8f16: return fastEmit_ISD_FMUL_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11785 case MVT::nxv8bf16: return fastEmit_ISD_FMUL_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11786 case MVT::nxv4f32: return fastEmit_ISD_FMUL_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11787 case MVT::nxv2f64: return fastEmit_ISD_FMUL_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11788 default: return 0;
11789 }
11790}
11791
11792// FastEmit functions for ISD::FSUB.
11793
11794unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11795 if (RetVT.SimpleTy != MVT::f16)
11796 return 0;
11797 if ((Subtarget->hasFullFP16())) {
11798 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
11799 }
11800 return 0;
11801}
11802
11803unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11804 if (RetVT.SimpleTy != MVT::f32)
11805 return 0;
11806 if ((Subtarget->hasFPARMv8())) {
11807 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
11808 }
11809 return 0;
11810}
11811
11812unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11813 if (RetVT.SimpleTy != MVT::f64)
11814 return 0;
11815 if ((Subtarget->hasFPARMv8())) {
11816 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
11817 }
11818 return 0;
11819}
11820
11821unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11822 if (RetVT.SimpleTy != MVT::v4f16)
11823 return 0;
11824 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11825 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11826 }
11827 return 0;
11828}
11829
11830unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11831 if (RetVT.SimpleTy != MVT::v8f16)
11832 return 0;
11833 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
11834 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11835 }
11836 return 0;
11837}
11838
11839unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11840 if (RetVT.SimpleTy != MVT::v2f32)
11841 return 0;
11842 if ((Subtarget->isNeonAvailable())) {
11843 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11844 }
11845 return 0;
11846}
11847
11848unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11849 if (RetVT.SimpleTy != MVT::v4f32)
11850 return 0;
11851 if ((Subtarget->isNeonAvailable())) {
11852 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11853 }
11854 return 0;
11855}
11856
11857unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11858 if (RetVT.SimpleTy != MVT::v2f64)
11859 return 0;
11860 if ((Subtarget->isNeonAvailable())) {
11861 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
11862 }
11863 return 0;
11864}
11865
11866unsigned fastEmit_ISD_FSUB_MVT_nxv8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11867 if (RetVT.SimpleTy != MVT::nxv8f16)
11868 return 0;
11869 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11870 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
11871 }
11872 return 0;
11873}
11874
11875unsigned fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11876 if (RetVT.SimpleTy != MVT::nxv8bf16)
11877 return 0;
11878 if ((Subtarget->hasB16B16()) && (Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2()))) {
11879 return fastEmitInst_rr(MachineInstOpcode: AArch64::BFSUB_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
11880 }
11881 return 0;
11882}
11883
11884unsigned fastEmit_ISD_FSUB_MVT_nxv4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11885 if (RetVT.SimpleTy != MVT::nxv4f32)
11886 return 0;
11887 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11888 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
11889 }
11890 return 0;
11891}
11892
11893unsigned fastEmit_ISD_FSUB_MVT_nxv2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11894 if (RetVT.SimpleTy != MVT::nxv2f64)
11895 return 0;
11896 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
11897 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
11898 }
11899 return 0;
11900}
11901
11902unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11903 switch (VT.SimpleTy) {
11904 case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
11905 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
11906 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
11907 case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
11908 case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
11909 case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
11910 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
11911 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
11912 case MVT::nxv8f16: return fastEmit_ISD_FSUB_MVT_nxv8f16_rr(RetVT, Op0, Op1);
11913 case MVT::nxv8bf16: return fastEmit_ISD_FSUB_MVT_nxv8bf16_rr(RetVT, Op0, Op1);
11914 case MVT::nxv4f32: return fastEmit_ISD_FSUB_MVT_nxv4f32_rr(RetVT, Op0, Op1);
11915 case MVT::nxv2f64: return fastEmit_ISD_FSUB_MVT_nxv2f64_rr(RetVT, Op0, Op1);
11916 default: return 0;
11917 }
11918}
11919
11920// FastEmit functions for ISD::MUL.
11921
11922unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11923 if (RetVT.SimpleTy != MVT::v8i8)
11924 return 0;
11925 if ((Subtarget->isNeonAvailable())) {
11926 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
11927 }
11928 return 0;
11929}
11930
11931unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11932 if (RetVT.SimpleTy != MVT::v16i8)
11933 return 0;
11934 if ((Subtarget->isNeonAvailable())) {
11935 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
11936 }
11937 return 0;
11938}
11939
11940unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11941 if (RetVT.SimpleTy != MVT::v4i16)
11942 return 0;
11943 if ((Subtarget->isNeonAvailable())) {
11944 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
11945 }
11946 return 0;
11947}
11948
11949unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11950 if (RetVT.SimpleTy != MVT::v8i16)
11951 return 0;
11952 if ((Subtarget->isNeonAvailable())) {
11953 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
11954 }
11955 return 0;
11956}
11957
11958unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11959 if (RetVT.SimpleTy != MVT::v2i32)
11960 return 0;
11961 if ((Subtarget->isNeonAvailable())) {
11962 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
11963 }
11964 return 0;
11965}
11966
11967unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11968 if (RetVT.SimpleTy != MVT::v4i32)
11969 return 0;
11970 if ((Subtarget->isNeonAvailable())) {
11971 return fastEmitInst_rr(MachineInstOpcode: AArch64::MULv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
11972 }
11973 return 0;
11974}
11975
11976unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11977 switch (VT.SimpleTy) {
11978 case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op1);
11979 case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op1);
11980 case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op1);
11981 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op1);
11982 case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op1);
11983 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op1);
11984 default: return 0;
11985 }
11986}
11987
11988// FastEmit functions for ISD::MULHS.
11989
11990unsigned fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
11991 if (RetVT.SimpleTy != MVT::i64)
11992 return 0;
11993 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
11994}
11995
11996unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
11997 switch (VT.SimpleTy) {
11998 case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op1);
11999 default: return 0;
12000 }
12001}
12002
12003// FastEmit functions for ISD::MULHU.
12004
12005unsigned fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12006 if (RetVT.SimpleTy != MVT::i64)
12007 return 0;
12008 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMULHrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12009}
12010
12011unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12012 switch (VT.SimpleTy) {
12013 case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op1);
12014 default: return 0;
12015 }
12016}
12017
12018// FastEmit functions for ISD::OR.
12019
12020unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12021 if (RetVT.SimpleTy != MVT::i32)
12022 return 0;
12023 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12024}
12025
12026unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12027 if (RetVT.SimpleTy != MVT::i64)
12028 return 0;
12029 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12030}
12031
12032unsigned fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12033 if (RetVT.SimpleTy != MVT::v8i8)
12034 return 0;
12035 if ((Subtarget->isNeonAvailable())) {
12036 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12037 }
12038 return 0;
12039}
12040
12041unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12042 if (RetVT.SimpleTy != MVT::v16i8)
12043 return 0;
12044 if ((Subtarget->isNeonAvailable())) {
12045 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12046 }
12047 return 0;
12048}
12049
12050unsigned fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12051 if (RetVT.SimpleTy != MVT::v4i16)
12052 return 0;
12053 if ((Subtarget->isNeonAvailable())) {
12054 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12055 }
12056 return 0;
12057}
12058
12059unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12060 if (RetVT.SimpleTy != MVT::v8i16)
12061 return 0;
12062 if ((Subtarget->isNeonAvailable())) {
12063 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12064 }
12065 return 0;
12066}
12067
12068unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12069 if (RetVT.SimpleTy != MVT::v2i32)
12070 return 0;
12071 if ((Subtarget->isNeonAvailable())) {
12072 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12073 }
12074 return 0;
12075}
12076
12077unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12078 if (RetVT.SimpleTy != MVT::v4i32)
12079 return 0;
12080 if ((Subtarget->isNeonAvailable())) {
12081 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12082 }
12083 return 0;
12084}
12085
12086unsigned fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12087 if (RetVT.SimpleTy != MVT::v1i64)
12088 return 0;
12089 if ((Subtarget->isNeonAvailable())) {
12090 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12091 }
12092 return 0;
12093}
12094
12095unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12096 if (RetVT.SimpleTy != MVT::v2i64)
12097 return 0;
12098 if ((Subtarget->isNeonAvailable())) {
12099 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORRv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12100 }
12101 return 0;
12102}
12103
12104unsigned fastEmit_ISD_OR_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12105 if (RetVT.SimpleTy != MVT::nxv16i8)
12106 return 0;
12107 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12108 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12109 }
12110 return 0;
12111}
12112
12113unsigned fastEmit_ISD_OR_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12114 if (RetVT.SimpleTy != MVT::nxv8i16)
12115 return 0;
12116 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12117 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12118 }
12119 return 0;
12120}
12121
12122unsigned fastEmit_ISD_OR_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12123 if (RetVT.SimpleTy != MVT::nxv4i32)
12124 return 0;
12125 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12126 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12127 }
12128 return 0;
12129}
12130
12131unsigned fastEmit_ISD_OR_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12132 if (RetVT.SimpleTy != MVT::nxv2i64)
12133 return 0;
12134 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12135 return fastEmitInst_rr(MachineInstOpcode: AArch64::ORR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
12136 }
12137 return 0;
12138}
12139
12140unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12141 switch (VT.SimpleTy) {
12142 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op1);
12143 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op1);
12144 case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op1);
12145 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op1);
12146 case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op1);
12147 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op1);
12148 case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op1);
12149 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op1);
12150 case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op1);
12151 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op1);
12152 case MVT::nxv16i8: return fastEmit_ISD_OR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12153 case MVT::nxv8i16: return fastEmit_ISD_OR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12154 case MVT::nxv4i32: return fastEmit_ISD_OR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12155 case MVT::nxv2i64: return fastEmit_ISD_OR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12156 default: return 0;
12157 }
12158}
12159
12160// FastEmit functions for ISD::ROTR.
12161
12162unsigned fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12163 if (RetVT.SimpleTy != MVT::i64)
12164 return 0;
12165 return fastEmitInst_rr(MachineInstOpcode: AArch64::RORVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12166}
12167
12168unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12169 switch (VT.SimpleTy) {
12170 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op1);
12171 default: return 0;
12172 }
12173}
12174
12175// FastEmit functions for ISD::SADDSAT.
12176
12177unsigned fastEmit_ISD_SADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12178 if (RetVT.SimpleTy != MVT::v8i8)
12179 return 0;
12180 if ((Subtarget->isNeonAvailable())) {
12181 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12182 }
12183 return 0;
12184}
12185
12186unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12187 if (RetVT.SimpleTy != MVT::v16i8)
12188 return 0;
12189 if ((Subtarget->isNeonAvailable())) {
12190 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12191 }
12192 return 0;
12193}
12194
12195unsigned fastEmit_ISD_SADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12196 if (RetVT.SimpleTy != MVT::v4i16)
12197 return 0;
12198 if ((Subtarget->isNeonAvailable())) {
12199 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12200 }
12201 return 0;
12202}
12203
12204unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12205 if (RetVT.SimpleTy != MVT::v8i16)
12206 return 0;
12207 if ((Subtarget->isNeonAvailable())) {
12208 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12209 }
12210 return 0;
12211}
12212
12213unsigned fastEmit_ISD_SADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12214 if (RetVT.SimpleTy != MVT::v2i32)
12215 return 0;
12216 if ((Subtarget->isNeonAvailable())) {
12217 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12218 }
12219 return 0;
12220}
12221
12222unsigned fastEmit_ISD_SADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12223 if (RetVT.SimpleTy != MVT::v4i32)
12224 return 0;
12225 if ((Subtarget->isNeonAvailable())) {
12226 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12227 }
12228 return 0;
12229}
12230
12231unsigned fastEmit_ISD_SADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12232 if (RetVT.SimpleTy != MVT::v2i64)
12233 return 0;
12234 if ((Subtarget->isNeonAvailable())) {
12235 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12236 }
12237 return 0;
12238}
12239
12240unsigned fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12241 if (RetVT.SimpleTy != MVT::nxv16i8)
12242 return 0;
12243 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12244 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12245 }
12246 return 0;
12247}
12248
12249unsigned fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12250 if (RetVT.SimpleTy != MVT::nxv8i16)
12251 return 0;
12252 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12253 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12254 }
12255 return 0;
12256}
12257
12258unsigned fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12259 if (RetVT.SimpleTy != MVT::nxv4i32)
12260 return 0;
12261 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12262 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12263 }
12264 return 0;
12265}
12266
12267unsigned fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12268 if (RetVT.SimpleTy != MVT::nxv2i64)
12269 return 0;
12270 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12271 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12272 }
12273 return 0;
12274}
12275
12276unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12277 switch (VT.SimpleTy) {
12278 case MVT::v8i8: return fastEmit_ISD_SADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12279 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12280 case MVT::v4i16: return fastEmit_ISD_SADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12281 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12282 case MVT::v2i32: return fastEmit_ISD_SADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12283 case MVT::v4i32: return fastEmit_ISD_SADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12284 case MVT::v2i64: return fastEmit_ISD_SADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12285 case MVT::nxv16i8: return fastEmit_ISD_SADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12286 case MVT::nxv8i16: return fastEmit_ISD_SADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12287 case MVT::nxv4i32: return fastEmit_ISD_SADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12288 case MVT::nxv2i64: return fastEmit_ISD_SADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12289 default: return 0;
12290 }
12291}
12292
12293// FastEmit functions for ISD::SDIV.
12294
12295unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12296 if (RetVT.SimpleTy != MVT::i32)
12297 return 0;
12298 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12299}
12300
12301unsigned fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12302 if (RetVT.SimpleTy != MVT::i64)
12303 return 0;
12304 return fastEmitInst_rr(MachineInstOpcode: AArch64::SDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12305}
12306
12307unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12308 switch (VT.SimpleTy) {
12309 case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op1);
12310 case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op1);
12311 default: return 0;
12312 }
12313}
12314
12315// FastEmit functions for ISD::SHL.
12316
12317unsigned fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12318 if (RetVT.SimpleTy != MVT::i64)
12319 return 0;
12320 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSLVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12321}
12322
12323unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12324 switch (VT.SimpleTy) {
12325 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op1);
12326 default: return 0;
12327 }
12328}
12329
12330// FastEmit functions for ISD::SMAX.
12331
12332unsigned fastEmit_ISD_SMAX_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12333 if (RetVT.SimpleTy != MVT::i32)
12334 return 0;
12335 if ((Subtarget->hasCSSC())) {
12336 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12337 }
12338 return 0;
12339}
12340
12341unsigned fastEmit_ISD_SMAX_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12342 if (RetVT.SimpleTy != MVT::i64)
12343 return 0;
12344 if ((Subtarget->hasCSSC())) {
12345 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12346 }
12347 return 0;
12348}
12349
12350unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12351 if (RetVT.SimpleTy != MVT::v8i8)
12352 return 0;
12353 if ((Subtarget->isNeonAvailable())) {
12354 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12355 }
12356 return 0;
12357}
12358
12359unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12360 if (RetVT.SimpleTy != MVT::v16i8)
12361 return 0;
12362 if ((Subtarget->isNeonAvailable())) {
12363 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12364 }
12365 return 0;
12366}
12367
12368unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12369 if (RetVT.SimpleTy != MVT::v4i16)
12370 return 0;
12371 if ((Subtarget->isNeonAvailable())) {
12372 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12373 }
12374 return 0;
12375}
12376
12377unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12378 if (RetVT.SimpleTy != MVT::v8i16)
12379 return 0;
12380 if ((Subtarget->isNeonAvailable())) {
12381 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12382 }
12383 return 0;
12384}
12385
12386unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12387 if (RetVT.SimpleTy != MVT::v2i32)
12388 return 0;
12389 if ((Subtarget->isNeonAvailable())) {
12390 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12391 }
12392 return 0;
12393}
12394
12395unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12396 if (RetVT.SimpleTy != MVT::v4i32)
12397 return 0;
12398 if ((Subtarget->isNeonAvailable())) {
12399 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12400 }
12401 return 0;
12402}
12403
12404unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12405 switch (VT.SimpleTy) {
12406 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_rr(RetVT, Op0, Op1);
12407 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_rr(RetVT, Op0, Op1);
12408 case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
12409 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
12410 case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
12411 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
12412 case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
12413 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
12414 default: return 0;
12415 }
12416}
12417
12418// FastEmit functions for ISD::SMIN.
12419
12420unsigned fastEmit_ISD_SMIN_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12421 if (RetVT.SimpleTy != MVT::i32)
12422 return 0;
12423 if ((Subtarget->hasCSSC())) {
12424 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
12425 }
12426 return 0;
12427}
12428
12429unsigned fastEmit_ISD_SMIN_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12430 if (RetVT.SimpleTy != MVT::i64)
12431 return 0;
12432 if ((Subtarget->hasCSSC())) {
12433 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12434 }
12435 return 0;
12436}
12437
12438unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12439 if (RetVT.SimpleTy != MVT::v8i8)
12440 return 0;
12441 if ((Subtarget->isNeonAvailable())) {
12442 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12443 }
12444 return 0;
12445}
12446
12447unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12448 if (RetVT.SimpleTy != MVT::v16i8)
12449 return 0;
12450 if ((Subtarget->isNeonAvailable())) {
12451 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12452 }
12453 return 0;
12454}
12455
12456unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12457 if (RetVT.SimpleTy != MVT::v4i16)
12458 return 0;
12459 if ((Subtarget->isNeonAvailable())) {
12460 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12461 }
12462 return 0;
12463}
12464
12465unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12466 if (RetVT.SimpleTy != MVT::v8i16)
12467 return 0;
12468 if ((Subtarget->isNeonAvailable())) {
12469 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12470 }
12471 return 0;
12472}
12473
12474unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12475 if (RetVT.SimpleTy != MVT::v2i32)
12476 return 0;
12477 if ((Subtarget->isNeonAvailable())) {
12478 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12479 }
12480 return 0;
12481}
12482
12483unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12484 if (RetVT.SimpleTy != MVT::v4i32)
12485 return 0;
12486 if ((Subtarget->isNeonAvailable())) {
12487 return fastEmitInst_rr(MachineInstOpcode: AArch64::SMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12488 }
12489 return 0;
12490}
12491
12492unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12493 switch (VT.SimpleTy) {
12494 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_rr(RetVT, Op0, Op1);
12495 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_rr(RetVT, Op0, Op1);
12496 case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
12497 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
12498 case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
12499 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
12500 case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
12501 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
12502 default: return 0;
12503 }
12504}
12505
12506// FastEmit functions for ISD::SRA.
12507
12508unsigned fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12509 if (RetVT.SimpleTy != MVT::i64)
12510 return 0;
12511 return fastEmitInst_rr(MachineInstOpcode: AArch64::ASRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12512}
12513
12514unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12515 switch (VT.SimpleTy) {
12516 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op1);
12517 default: return 0;
12518 }
12519}
12520
12521// FastEmit functions for ISD::SRL.
12522
12523unsigned fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12524 if (RetVT.SimpleTy != MVT::i64)
12525 return 0;
12526 return fastEmitInst_rr(MachineInstOpcode: AArch64::LSRVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
12527}
12528
12529unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12530 switch (VT.SimpleTy) {
12531 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op1);
12532 default: return 0;
12533 }
12534}
12535
12536// FastEmit functions for ISD::SSUBSAT.
12537
12538unsigned fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12539 if (RetVT.SimpleTy != MVT::v8i8)
12540 return 0;
12541 if ((Subtarget->isNeonAvailable())) {
12542 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
12543 }
12544 return 0;
12545}
12546
12547unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12548 if (RetVT.SimpleTy != MVT::v16i8)
12549 return 0;
12550 if ((Subtarget->isNeonAvailable())) {
12551 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
12552 }
12553 return 0;
12554}
12555
12556unsigned fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12557 if (RetVT.SimpleTy != MVT::v4i16)
12558 return 0;
12559 if ((Subtarget->isNeonAvailable())) {
12560 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12561 }
12562 return 0;
12563}
12564
12565unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12566 if (RetVT.SimpleTy != MVT::v8i16)
12567 return 0;
12568 if ((Subtarget->isNeonAvailable())) {
12569 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12570 }
12571 return 0;
12572}
12573
12574unsigned fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12575 if (RetVT.SimpleTy != MVT::v2i32)
12576 return 0;
12577 if ((Subtarget->isNeonAvailable())) {
12578 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12579 }
12580 return 0;
12581}
12582
12583unsigned fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12584 if (RetVT.SimpleTy != MVT::v4i32)
12585 return 0;
12586 if ((Subtarget->isNeonAvailable())) {
12587 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12588 }
12589 return 0;
12590}
12591
12592unsigned fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12593 if (RetVT.SimpleTy != MVT::v2i64)
12594 return 0;
12595 if ((Subtarget->isNeonAvailable())) {
12596 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12597 }
12598 return 0;
12599}
12600
12601unsigned fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12602 if (RetVT.SimpleTy != MVT::nxv16i8)
12603 return 0;
12604 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12605 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
12606 }
12607 return 0;
12608}
12609
12610unsigned fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12611 if (RetVT.SimpleTy != MVT::nxv8i16)
12612 return 0;
12613 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12614 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
12615 }
12616 return 0;
12617}
12618
12619unsigned fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12620 if (RetVT.SimpleTy != MVT::nxv4i32)
12621 return 0;
12622 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12623 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
12624 }
12625 return 0;
12626}
12627
12628unsigned fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12629 if (RetVT.SimpleTy != MVT::nxv2i64)
12630 return 0;
12631 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
12632 return fastEmitInst_rr(MachineInstOpcode: AArch64::SQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
12633 }
12634 return 0;
12635}
12636
12637unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12638 switch (VT.SimpleTy) {
12639 case MVT::v8i8: return fastEmit_ISD_SSUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
12640 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
12641 case MVT::v4i16: return fastEmit_ISD_SSUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
12642 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
12643 case MVT::v2i32: return fastEmit_ISD_SSUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
12644 case MVT::v4i32: return fastEmit_ISD_SSUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
12645 case MVT::v2i64: return fastEmit_ISD_SSUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
12646 case MVT::nxv16i8: return fastEmit_ISD_SSUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
12647 case MVT::nxv8i16: return fastEmit_ISD_SSUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
12648 case MVT::nxv4i32: return fastEmit_ISD_SSUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
12649 case MVT::nxv2i64: return fastEmit_ISD_SSUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
12650 default: return 0;
12651 }
12652}
12653
12654// FastEmit functions for ISD::STRICT_FADD.
12655
12656unsigned fastEmit_ISD_STRICT_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12657 if (RetVT.SimpleTy != MVT::f16)
12658 return 0;
12659 if ((Subtarget->hasFullFP16())) {
12660 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12661 }
12662 return 0;
12663}
12664
12665unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12666 if (RetVT.SimpleTy != MVT::f32)
12667 return 0;
12668 if ((Subtarget->hasFPARMv8())) {
12669 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12670 }
12671 return 0;
12672}
12673
12674unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12675 if (RetVT.SimpleTy != MVT::f64)
12676 return 0;
12677 if ((Subtarget->hasFPARMv8())) {
12678 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12679 }
12680 return 0;
12681}
12682
12683unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12684 if (RetVT.SimpleTy != MVT::v4f16)
12685 return 0;
12686 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12687 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12688 }
12689 return 0;
12690}
12691
12692unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12693 if (RetVT.SimpleTy != MVT::v8f16)
12694 return 0;
12695 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12696 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12697 }
12698 return 0;
12699}
12700
12701unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12702 if (RetVT.SimpleTy != MVT::v2f32)
12703 return 0;
12704 if ((Subtarget->isNeonAvailable())) {
12705 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12706 }
12707 return 0;
12708}
12709
12710unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12711 if (RetVT.SimpleTy != MVT::v4f32)
12712 return 0;
12713 if ((Subtarget->isNeonAvailable())) {
12714 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12715 }
12716 return 0;
12717}
12718
12719unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12720 if (RetVT.SimpleTy != MVT::v2f64)
12721 return 0;
12722 if ((Subtarget->isNeonAvailable())) {
12723 return fastEmitInst_rr(MachineInstOpcode: AArch64::FADDv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12724 }
12725 return 0;
12726}
12727
12728unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12729 switch (VT.SimpleTy) {
12730 case MVT::f16: return fastEmit_ISD_STRICT_FADD_MVT_f16_rr(RetVT, Op0, Op1);
12731 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op1);
12732 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op1);
12733 case MVT::v4f16: return fastEmit_ISD_STRICT_FADD_MVT_v4f16_rr(RetVT, Op0, Op1);
12734 case MVT::v8f16: return fastEmit_ISD_STRICT_FADD_MVT_v8f16_rr(RetVT, Op0, Op1);
12735 case MVT::v2f32: return fastEmit_ISD_STRICT_FADD_MVT_v2f32_rr(RetVT, Op0, Op1);
12736 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op1);
12737 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op1);
12738 default: return 0;
12739 }
12740}
12741
12742// FastEmit functions for ISD::STRICT_FDIV.
12743
12744unsigned fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12745 if (RetVT.SimpleTy != MVT::f16)
12746 return 0;
12747 if ((Subtarget->hasFullFP16())) {
12748 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12749 }
12750 return 0;
12751}
12752
12753unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12754 if (RetVT.SimpleTy != MVT::f32)
12755 return 0;
12756 if ((Subtarget->hasFPARMv8())) {
12757 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12758 }
12759 return 0;
12760}
12761
12762unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12763 if (RetVT.SimpleTy != MVT::f64)
12764 return 0;
12765 if ((Subtarget->hasFPARMv8())) {
12766 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12767 }
12768 return 0;
12769}
12770
12771unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12772 if (RetVT.SimpleTy != MVT::v4f16)
12773 return 0;
12774 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12775 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12776 }
12777 return 0;
12778}
12779
12780unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12781 if (RetVT.SimpleTy != MVT::v8f16)
12782 return 0;
12783 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12784 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12785 }
12786 return 0;
12787}
12788
12789unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12790 if (RetVT.SimpleTy != MVT::v2f32)
12791 return 0;
12792 if ((Subtarget->isNeonAvailable())) {
12793 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12794 }
12795 return 0;
12796}
12797
12798unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12799 if (RetVT.SimpleTy != MVT::v4f32)
12800 return 0;
12801 if ((Subtarget->isNeonAvailable())) {
12802 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12803 }
12804 return 0;
12805}
12806
12807unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12808 if (RetVT.SimpleTy != MVT::v2f64)
12809 return 0;
12810 if ((Subtarget->isNeonAvailable())) {
12811 return fastEmitInst_rr(MachineInstOpcode: AArch64::FDIVv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12812 }
12813 return 0;
12814}
12815
12816unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12817 switch (VT.SimpleTy) {
12818 case MVT::f16: return fastEmit_ISD_STRICT_FDIV_MVT_f16_rr(RetVT, Op0, Op1);
12819 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op1);
12820 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op1);
12821 case MVT::v4f16: return fastEmit_ISD_STRICT_FDIV_MVT_v4f16_rr(RetVT, Op0, Op1);
12822 case MVT::v8f16: return fastEmit_ISD_STRICT_FDIV_MVT_v8f16_rr(RetVT, Op0, Op1);
12823 case MVT::v2f32: return fastEmit_ISD_STRICT_FDIV_MVT_v2f32_rr(RetVT, Op0, Op1);
12824 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op1);
12825 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op1);
12826 default: return 0;
12827 }
12828}
12829
12830// FastEmit functions for ISD::STRICT_FMAXIMUM.
12831
12832unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12833 if (RetVT.SimpleTy != MVT::f16)
12834 return 0;
12835 if ((Subtarget->hasFullFP16())) {
12836 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12837 }
12838 return 0;
12839}
12840
12841unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12842 if (RetVT.SimpleTy != MVT::f32)
12843 return 0;
12844 if ((Subtarget->hasFPARMv8())) {
12845 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12846 }
12847 return 0;
12848}
12849
12850unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12851 if (RetVT.SimpleTy != MVT::f64)
12852 return 0;
12853 if ((Subtarget->hasFPARMv8())) {
12854 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12855 }
12856 return 0;
12857}
12858
12859unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12860 if (RetVT.SimpleTy != MVT::v4f16)
12861 return 0;
12862 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12863 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12864 }
12865 return 0;
12866}
12867
12868unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12869 if (RetVT.SimpleTy != MVT::v8f16)
12870 return 0;
12871 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12872 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12873 }
12874 return 0;
12875}
12876
12877unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12878 if (RetVT.SimpleTy != MVT::v2f32)
12879 return 0;
12880 if ((Subtarget->isNeonAvailable())) {
12881 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12882 }
12883 return 0;
12884}
12885
12886unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12887 if (RetVT.SimpleTy != MVT::v4f32)
12888 return 0;
12889 if ((Subtarget->isNeonAvailable())) {
12890 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12891 }
12892 return 0;
12893}
12894
12895unsigned fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12896 if (RetVT.SimpleTy != MVT::v2f64)
12897 return 0;
12898 if ((Subtarget->isNeonAvailable())) {
12899 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12900 }
12901 return 0;
12902}
12903
12904unsigned fastEmit_ISD_STRICT_FMAXIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12905 switch (VT.SimpleTy) {
12906 case MVT::f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f16_rr(RetVT, Op0, Op1);
12907 case MVT::f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f32_rr(RetVT, Op0, Op1);
12908 case MVT::f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_f64_rr(RetVT, Op0, Op1);
12909 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12910 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12911 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
12912 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
12913 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
12914 default: return 0;
12915 }
12916}
12917
12918// FastEmit functions for ISD::STRICT_FMAXNUM.
12919
12920unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12921 if (RetVT.SimpleTy != MVT::f16)
12922 return 0;
12923 if ((Subtarget->hasFullFP16())) {
12924 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
12925 }
12926 return 0;
12927}
12928
12929unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12930 if (RetVT.SimpleTy != MVT::f32)
12931 return 0;
12932 if ((Subtarget->hasFPARMv8())) {
12933 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
12934 }
12935 return 0;
12936}
12937
12938unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12939 if (RetVT.SimpleTy != MVT::f64)
12940 return 0;
12941 if ((Subtarget->hasFPARMv8())) {
12942 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
12943 }
12944 return 0;
12945}
12946
12947unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12948 if (RetVT.SimpleTy != MVT::v4f16)
12949 return 0;
12950 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12951 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
12952 }
12953 return 0;
12954}
12955
12956unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12957 if (RetVT.SimpleTy != MVT::v8f16)
12958 return 0;
12959 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
12960 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
12961 }
12962 return 0;
12963}
12964
12965unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12966 if (RetVT.SimpleTy != MVT::v2f32)
12967 return 0;
12968 if ((Subtarget->isNeonAvailable())) {
12969 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
12970 }
12971 return 0;
12972}
12973
12974unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12975 if (RetVT.SimpleTy != MVT::v4f32)
12976 return 0;
12977 if ((Subtarget->isNeonAvailable())) {
12978 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
12979 }
12980 return 0;
12981}
12982
12983unsigned fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
12984 if (RetVT.SimpleTy != MVT::v2f64)
12985 return 0;
12986 if ((Subtarget->isNeonAvailable())) {
12987 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMAXNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
12988 }
12989 return 0;
12990}
12991
12992unsigned fastEmit_ISD_STRICT_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
12993 switch (VT.SimpleTy) {
12994 case MVT::f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op1);
12995 case MVT::f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op1);
12996 case MVT::f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op1);
12997 case MVT::v4f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
12998 case MVT::v8f16: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
12999 case MVT::v2f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13000 case MVT::v4f32: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13001 case MVT::v2f64: return fastEmit_ISD_STRICT_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13002 default: return 0;
13003 }
13004}
13005
13006// FastEmit functions for ISD::STRICT_FMINIMUM.
13007
13008unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13009 if (RetVT.SimpleTy != MVT::f16)
13010 return 0;
13011 if ((Subtarget->hasFullFP16())) {
13012 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13013 }
13014 return 0;
13015}
13016
13017unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13018 if (RetVT.SimpleTy != MVT::f32)
13019 return 0;
13020 if ((Subtarget->hasFPARMv8())) {
13021 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13022 }
13023 return 0;
13024}
13025
13026unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13027 if (RetVT.SimpleTy != MVT::f64)
13028 return 0;
13029 if ((Subtarget->hasFPARMv8())) {
13030 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13031 }
13032 return 0;
13033}
13034
13035unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13036 if (RetVT.SimpleTy != MVT::v4f16)
13037 return 0;
13038 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13039 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13040 }
13041 return 0;
13042}
13043
13044unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13045 if (RetVT.SimpleTy != MVT::v8f16)
13046 return 0;
13047 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13048 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13049 }
13050 return 0;
13051}
13052
13053unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13054 if (RetVT.SimpleTy != MVT::v2f32)
13055 return 0;
13056 if ((Subtarget->isNeonAvailable())) {
13057 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13058 }
13059 return 0;
13060}
13061
13062unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13063 if (RetVT.SimpleTy != MVT::v4f32)
13064 return 0;
13065 if ((Subtarget->isNeonAvailable())) {
13066 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13067 }
13068 return 0;
13069}
13070
13071unsigned fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13072 if (RetVT.SimpleTy != MVT::v2f64)
13073 return 0;
13074 if ((Subtarget->isNeonAvailable())) {
13075 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13076 }
13077 return 0;
13078}
13079
13080unsigned fastEmit_ISD_STRICT_FMINIMUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13081 switch (VT.SimpleTy) {
13082 case MVT::f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f16_rr(RetVT, Op0, Op1);
13083 case MVT::f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f32_rr(RetVT, Op0, Op1);
13084 case MVT::f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_f64_rr(RetVT, Op0, Op1);
13085 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13086 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13087 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13088 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13089 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINIMUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13090 default: return 0;
13091 }
13092}
13093
13094// FastEmit functions for ISD::STRICT_FMINNUM.
13095
13096unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13097 if (RetVT.SimpleTy != MVT::f16)
13098 return 0;
13099 if ((Subtarget->hasFullFP16())) {
13100 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13101 }
13102 return 0;
13103}
13104
13105unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13106 if (RetVT.SimpleTy != MVT::f32)
13107 return 0;
13108 if ((Subtarget->hasFPARMv8())) {
13109 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13110 }
13111 return 0;
13112}
13113
13114unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13115 if (RetVT.SimpleTy != MVT::f64)
13116 return 0;
13117 if ((Subtarget->hasFPARMv8())) {
13118 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13119 }
13120 return 0;
13121}
13122
13123unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13124 if (RetVT.SimpleTy != MVT::v4f16)
13125 return 0;
13126 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13127 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13128 }
13129 return 0;
13130}
13131
13132unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13133 if (RetVT.SimpleTy != MVT::v8f16)
13134 return 0;
13135 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13136 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13137 }
13138 return 0;
13139}
13140
13141unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13142 if (RetVT.SimpleTy != MVT::v2f32)
13143 return 0;
13144 if ((Subtarget->isNeonAvailable())) {
13145 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13146 }
13147 return 0;
13148}
13149
13150unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13151 if (RetVT.SimpleTy != MVT::v4f32)
13152 return 0;
13153 if ((Subtarget->isNeonAvailable())) {
13154 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13155 }
13156 return 0;
13157}
13158
13159unsigned fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13160 if (RetVT.SimpleTy != MVT::v2f64)
13161 return 0;
13162 if ((Subtarget->isNeonAvailable())) {
13163 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMINNMv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13164 }
13165 return 0;
13166}
13167
13168unsigned fastEmit_ISD_STRICT_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13169 switch (VT.SimpleTy) {
13170 case MVT::f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_f16_rr(RetVT, Op0, Op1);
13171 case MVT::f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_f32_rr(RetVT, Op0, Op1);
13172 case MVT::f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_f64_rr(RetVT, Op0, Op1);
13173 case MVT::v4f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op1);
13174 case MVT::v8f16: return fastEmit_ISD_STRICT_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op1);
13175 case MVT::v2f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op1);
13176 case MVT::v4f32: return fastEmit_ISD_STRICT_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op1);
13177 case MVT::v2f64: return fastEmit_ISD_STRICT_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op1);
13178 default: return 0;
13179 }
13180}
13181
13182// FastEmit functions for ISD::STRICT_FMUL.
13183
13184unsigned fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13185 if (RetVT.SimpleTy != MVT::f16)
13186 return 0;
13187 if ((Subtarget->hasFullFP16())) {
13188 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13189 }
13190 return 0;
13191}
13192
13193unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13194 if (RetVT.SimpleTy != MVT::f32)
13195 return 0;
13196 if ((Subtarget->hasFPARMv8())) {
13197 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13198 }
13199 return 0;
13200}
13201
13202unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13203 if (RetVT.SimpleTy != MVT::f64)
13204 return 0;
13205 if ((Subtarget->hasFPARMv8())) {
13206 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13207 }
13208 return 0;
13209}
13210
13211unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13212 if (RetVT.SimpleTy != MVT::v4f16)
13213 return 0;
13214 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13215 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13216 }
13217 return 0;
13218}
13219
13220unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13221 if (RetVT.SimpleTy != MVT::v8f16)
13222 return 0;
13223 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13224 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13225 }
13226 return 0;
13227}
13228
13229unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13230 if (RetVT.SimpleTy != MVT::v2f32)
13231 return 0;
13232 if ((Subtarget->isNeonAvailable())) {
13233 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13234 }
13235 return 0;
13236}
13237
13238unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13239 if (RetVT.SimpleTy != MVT::v4f32)
13240 return 0;
13241 if ((Subtarget->isNeonAvailable())) {
13242 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13243 }
13244 return 0;
13245}
13246
13247unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13248 if (RetVT.SimpleTy != MVT::v2f64)
13249 return 0;
13250 if ((Subtarget->isNeonAvailable())) {
13251 return fastEmitInst_rr(MachineInstOpcode: AArch64::FMULv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13252 }
13253 return 0;
13254}
13255
13256unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13257 switch (VT.SimpleTy) {
13258 case MVT::f16: return fastEmit_ISD_STRICT_FMUL_MVT_f16_rr(RetVT, Op0, Op1);
13259 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op1);
13260 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op1);
13261 case MVT::v4f16: return fastEmit_ISD_STRICT_FMUL_MVT_v4f16_rr(RetVT, Op0, Op1);
13262 case MVT::v8f16: return fastEmit_ISD_STRICT_FMUL_MVT_v8f16_rr(RetVT, Op0, Op1);
13263 case MVT::v2f32: return fastEmit_ISD_STRICT_FMUL_MVT_v2f32_rr(RetVT, Op0, Op1);
13264 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op1);
13265 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op1);
13266 default: return 0;
13267 }
13268}
13269
13270// FastEmit functions for ISD::STRICT_FSUB.
13271
13272unsigned fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13273 if (RetVT.SimpleTy != MVT::f16)
13274 return 0;
13275 if ((Subtarget->hasFullFP16())) {
13276 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBHrr, RC: &AArch64::FPR16RegClass, Op0, Op1);
13277 }
13278 return 0;
13279}
13280
13281unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13282 if (RetVT.SimpleTy != MVT::f32)
13283 return 0;
13284 if ((Subtarget->hasFPARMv8())) {
13285 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBSrr, RC: &AArch64::FPR32RegClass, Op0, Op1);
13286 }
13287 return 0;
13288}
13289
13290unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13291 if (RetVT.SimpleTy != MVT::f64)
13292 return 0;
13293 if ((Subtarget->hasFPARMv8())) {
13294 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBDrr, RC: &AArch64::FPR64RegClass, Op0, Op1);
13295 }
13296 return 0;
13297}
13298
13299unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13300 if (RetVT.SimpleTy != MVT::v4f16)
13301 return 0;
13302 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13303 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13304 }
13305 return 0;
13306}
13307
13308unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13309 if (RetVT.SimpleTy != MVT::v8f16)
13310 return 0;
13311 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
13312 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv8f16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13313 }
13314 return 0;
13315}
13316
13317unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13318 if (RetVT.SimpleTy != MVT::v2f32)
13319 return 0;
13320 if ((Subtarget->isNeonAvailable())) {
13321 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13322 }
13323 return 0;
13324}
13325
13326unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13327 if (RetVT.SimpleTy != MVT::v4f32)
13328 return 0;
13329 if ((Subtarget->isNeonAvailable())) {
13330 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv4f32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13331 }
13332 return 0;
13333}
13334
13335unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13336 if (RetVT.SimpleTy != MVT::v2f64)
13337 return 0;
13338 if ((Subtarget->isNeonAvailable())) {
13339 return fastEmitInst_rr(MachineInstOpcode: AArch64::FSUBv2f64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13340 }
13341 return 0;
13342}
13343
13344unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13345 switch (VT.SimpleTy) {
13346 case MVT::f16: return fastEmit_ISD_STRICT_FSUB_MVT_f16_rr(RetVT, Op0, Op1);
13347 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op1);
13348 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op1);
13349 case MVT::v4f16: return fastEmit_ISD_STRICT_FSUB_MVT_v4f16_rr(RetVT, Op0, Op1);
13350 case MVT::v8f16: return fastEmit_ISD_STRICT_FSUB_MVT_v8f16_rr(RetVT, Op0, Op1);
13351 case MVT::v2f32: return fastEmit_ISD_STRICT_FSUB_MVT_v2f32_rr(RetVT, Op0, Op1);
13352 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op1);
13353 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op1);
13354 default: return 0;
13355 }
13356}
13357
13358// FastEmit functions for ISD::SUB.
13359
13360unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13361 if (RetVT.SimpleTy != MVT::i32)
13362 return 0;
13363 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13364}
13365
13366unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13367 if (RetVT.SimpleTy != MVT::i64)
13368 return 0;
13369 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBSXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13370}
13371
13372unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13373 if (RetVT.SimpleTy != MVT::v8i8)
13374 return 0;
13375 if ((Subtarget->isNeonAvailable())) {
13376 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13377 }
13378 return 0;
13379}
13380
13381unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13382 if (RetVT.SimpleTy != MVT::v16i8)
13383 return 0;
13384 if ((Subtarget->isNeonAvailable())) {
13385 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13386 }
13387 return 0;
13388}
13389
13390unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13391 if (RetVT.SimpleTy != MVT::v4i16)
13392 return 0;
13393 if ((Subtarget->isNeonAvailable())) {
13394 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13395 }
13396 return 0;
13397}
13398
13399unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13400 if (RetVT.SimpleTy != MVT::v8i16)
13401 return 0;
13402 if ((Subtarget->isNeonAvailable())) {
13403 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13404 }
13405 return 0;
13406}
13407
13408unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13409 if (RetVT.SimpleTy != MVT::v2i32)
13410 return 0;
13411 if ((Subtarget->isNeonAvailable())) {
13412 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13413 }
13414 return 0;
13415}
13416
13417unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13418 if (RetVT.SimpleTy != MVT::v4i32)
13419 return 0;
13420 if ((Subtarget->isNeonAvailable())) {
13421 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13422 }
13423 return 0;
13424}
13425
13426unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13427 if (RetVT.SimpleTy != MVT::v1i64)
13428 return 0;
13429 if ((Subtarget->isNeonAvailable())) {
13430 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv1i64, RC: &AArch64::FPR64RegClass, Op0, Op1);
13431 }
13432 return 0;
13433}
13434
13435unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13436 if (RetVT.SimpleTy != MVT::v2i64)
13437 return 0;
13438 if ((Subtarget->isNeonAvailable())) {
13439 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13440 }
13441 return 0;
13442}
13443
13444unsigned fastEmit_ISD_SUB_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13445 if (RetVT.SimpleTy != MVT::nxv16i8)
13446 return 0;
13447 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13448 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13449 }
13450 return 0;
13451}
13452
13453unsigned fastEmit_ISD_SUB_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13454 if (RetVT.SimpleTy != MVT::nxv8i16)
13455 return 0;
13456 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13457 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13458 }
13459 return 0;
13460}
13461
13462unsigned fastEmit_ISD_SUB_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13463 if (RetVT.SimpleTy != MVT::nxv4i32)
13464 return 0;
13465 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13466 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13467 }
13468 return 0;
13469}
13470
13471unsigned fastEmit_ISD_SUB_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13472 if (RetVT.SimpleTy != MVT::nxv2i64)
13473 return 0;
13474 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13475 return fastEmitInst_rr(MachineInstOpcode: AArch64::SUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13476 }
13477 return 0;
13478}
13479
13480unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13481 switch (VT.SimpleTy) {
13482 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op1);
13483 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op1);
13484 case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op1);
13485 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op1);
13486 case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op1);
13487 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op1);
13488 case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op1);
13489 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op1);
13490 case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op1);
13491 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op1);
13492 case MVT::nxv16i8: return fastEmit_ISD_SUB_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13493 case MVT::nxv8i16: return fastEmit_ISD_SUB_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13494 case MVT::nxv4i32: return fastEmit_ISD_SUB_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13495 case MVT::nxv2i64: return fastEmit_ISD_SUB_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13496 default: return 0;
13497 }
13498}
13499
13500// FastEmit functions for ISD::UADDSAT.
13501
13502unsigned fastEmit_ISD_UADDSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13503 if (RetVT.SimpleTy != MVT::v8i8)
13504 return 0;
13505 if ((Subtarget->isNeonAvailable())) {
13506 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13507 }
13508 return 0;
13509}
13510
13511unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13512 if (RetVT.SimpleTy != MVT::v16i8)
13513 return 0;
13514 if ((Subtarget->isNeonAvailable())) {
13515 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13516 }
13517 return 0;
13518}
13519
13520unsigned fastEmit_ISD_UADDSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13521 if (RetVT.SimpleTy != MVT::v4i16)
13522 return 0;
13523 if ((Subtarget->isNeonAvailable())) {
13524 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13525 }
13526 return 0;
13527}
13528
13529unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13530 if (RetVT.SimpleTy != MVT::v8i16)
13531 return 0;
13532 if ((Subtarget->isNeonAvailable())) {
13533 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13534 }
13535 return 0;
13536}
13537
13538unsigned fastEmit_ISD_UADDSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13539 if (RetVT.SimpleTy != MVT::v2i32)
13540 return 0;
13541 if ((Subtarget->isNeonAvailable())) {
13542 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13543 }
13544 return 0;
13545}
13546
13547unsigned fastEmit_ISD_UADDSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13548 if (RetVT.SimpleTy != MVT::v4i32)
13549 return 0;
13550 if ((Subtarget->isNeonAvailable())) {
13551 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13552 }
13553 return 0;
13554}
13555
13556unsigned fastEmit_ISD_UADDSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13557 if (RetVT.SimpleTy != MVT::v2i64)
13558 return 0;
13559 if ((Subtarget->isNeonAvailable())) {
13560 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADDv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13561 }
13562 return 0;
13563}
13564
13565unsigned fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13566 if (RetVT.SimpleTy != MVT::nxv16i8)
13567 return 0;
13568 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13569 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13570 }
13571 return 0;
13572}
13573
13574unsigned fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13575 if (RetVT.SimpleTy != MVT::nxv8i16)
13576 return 0;
13577 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13578 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13579 }
13580 return 0;
13581}
13582
13583unsigned fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13584 if (RetVT.SimpleTy != MVT::nxv4i32)
13585 return 0;
13586 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13587 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13588 }
13589 return 0;
13590}
13591
13592unsigned fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13593 if (RetVT.SimpleTy != MVT::nxv2i64)
13594 return 0;
13595 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13596 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQADD_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13597 }
13598 return 0;
13599}
13600
13601unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13602 switch (VT.SimpleTy) {
13603 case MVT::v8i8: return fastEmit_ISD_UADDSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13604 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13605 case MVT::v4i16: return fastEmit_ISD_UADDSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13606 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13607 case MVT::v2i32: return fastEmit_ISD_UADDSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13608 case MVT::v4i32: return fastEmit_ISD_UADDSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13609 case MVT::v2i64: return fastEmit_ISD_UADDSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13610 case MVT::nxv16i8: return fastEmit_ISD_UADDSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13611 case MVT::nxv8i16: return fastEmit_ISD_UADDSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13612 case MVT::nxv4i32: return fastEmit_ISD_UADDSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13613 case MVT::nxv2i64: return fastEmit_ISD_UADDSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13614 default: return 0;
13615 }
13616}
13617
13618// FastEmit functions for ISD::UDIV.
13619
13620unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13621 if (RetVT.SimpleTy != MVT::i32)
13622 return 0;
13623 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVWr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13624}
13625
13626unsigned fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13627 if (RetVT.SimpleTy != MVT::i64)
13628 return 0;
13629 return fastEmitInst_rr(MachineInstOpcode: AArch64::UDIVXr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13630}
13631
13632unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13633 switch (VT.SimpleTy) {
13634 case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op1);
13635 case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op1);
13636 default: return 0;
13637 }
13638}
13639
13640// FastEmit functions for ISD::UMAX.
13641
13642unsigned fastEmit_ISD_UMAX_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13643 if (RetVT.SimpleTy != MVT::i32)
13644 return 0;
13645 if ((Subtarget->hasCSSC())) {
13646 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13647 }
13648 return 0;
13649}
13650
13651unsigned fastEmit_ISD_UMAX_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13652 if (RetVT.SimpleTy != MVT::i64)
13653 return 0;
13654 if ((Subtarget->hasCSSC())) {
13655 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13656 }
13657 return 0;
13658}
13659
13660unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13661 if (RetVT.SimpleTy != MVT::v8i8)
13662 return 0;
13663 if ((Subtarget->isNeonAvailable())) {
13664 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13665 }
13666 return 0;
13667}
13668
13669unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13670 if (RetVT.SimpleTy != MVT::v16i8)
13671 return 0;
13672 if ((Subtarget->isNeonAvailable())) {
13673 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13674 }
13675 return 0;
13676}
13677
13678unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13679 if (RetVT.SimpleTy != MVT::v4i16)
13680 return 0;
13681 if ((Subtarget->isNeonAvailable())) {
13682 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13683 }
13684 return 0;
13685}
13686
13687unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13688 if (RetVT.SimpleTy != MVT::v8i16)
13689 return 0;
13690 if ((Subtarget->isNeonAvailable())) {
13691 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13692 }
13693 return 0;
13694}
13695
13696unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13697 if (RetVT.SimpleTy != MVT::v2i32)
13698 return 0;
13699 if ((Subtarget->isNeonAvailable())) {
13700 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13701 }
13702 return 0;
13703}
13704
13705unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13706 if (RetVT.SimpleTy != MVT::v4i32)
13707 return 0;
13708 if ((Subtarget->isNeonAvailable())) {
13709 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMAXv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13710 }
13711 return 0;
13712}
13713
13714unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13715 switch (VT.SimpleTy) {
13716 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_rr(RetVT, Op0, Op1);
13717 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_rr(RetVT, Op0, Op1);
13718 case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op1);
13719 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op1);
13720 case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op1);
13721 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op1);
13722 case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op1);
13723 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op1);
13724 default: return 0;
13725 }
13726}
13727
13728// FastEmit functions for ISD::UMIN.
13729
13730unsigned fastEmit_ISD_UMIN_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13731 if (RetVT.SimpleTy != MVT::i32)
13732 return 0;
13733 if ((Subtarget->hasCSSC())) {
13734 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13735 }
13736 return 0;
13737}
13738
13739unsigned fastEmit_ISD_UMIN_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13740 if (RetVT.SimpleTy != MVT::i64)
13741 return 0;
13742 if ((Subtarget->hasCSSC())) {
13743 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13744 }
13745 return 0;
13746}
13747
13748unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13749 if (RetVT.SimpleTy != MVT::v8i8)
13750 return 0;
13751 if ((Subtarget->isNeonAvailable())) {
13752 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13753 }
13754 return 0;
13755}
13756
13757unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13758 if (RetVT.SimpleTy != MVT::v16i8)
13759 return 0;
13760 if ((Subtarget->isNeonAvailable())) {
13761 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13762 }
13763 return 0;
13764}
13765
13766unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13767 if (RetVT.SimpleTy != MVT::v4i16)
13768 return 0;
13769 if ((Subtarget->isNeonAvailable())) {
13770 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13771 }
13772 return 0;
13773}
13774
13775unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13776 if (RetVT.SimpleTy != MVT::v8i16)
13777 return 0;
13778 if ((Subtarget->isNeonAvailable())) {
13779 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13780 }
13781 return 0;
13782}
13783
13784unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13785 if (RetVT.SimpleTy != MVT::v2i32)
13786 return 0;
13787 if ((Subtarget->isNeonAvailable())) {
13788 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13789 }
13790 return 0;
13791}
13792
13793unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13794 if (RetVT.SimpleTy != MVT::v4i32)
13795 return 0;
13796 if ((Subtarget->isNeonAvailable())) {
13797 return fastEmitInst_rr(MachineInstOpcode: AArch64::UMINv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13798 }
13799 return 0;
13800}
13801
13802unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13803 switch (VT.SimpleTy) {
13804 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_rr(RetVT, Op0, Op1);
13805 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_rr(RetVT, Op0, Op1);
13806 case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op1);
13807 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op1);
13808 case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op1);
13809 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op1);
13810 case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op1);
13811 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op1);
13812 default: return 0;
13813 }
13814}
13815
13816// FastEmit functions for ISD::USUBSAT.
13817
13818unsigned fastEmit_ISD_USUBSAT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13819 if (RetVT.SimpleTy != MVT::v8i8)
13820 return 0;
13821 if ((Subtarget->isNeonAvailable())) {
13822 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13823 }
13824 return 0;
13825}
13826
13827unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13828 if (RetVT.SimpleTy != MVT::v16i8)
13829 return 0;
13830 if ((Subtarget->isNeonAvailable())) {
13831 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13832 }
13833 return 0;
13834}
13835
13836unsigned fastEmit_ISD_USUBSAT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13837 if (RetVT.SimpleTy != MVT::v4i16)
13838 return 0;
13839 if ((Subtarget->isNeonAvailable())) {
13840 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i16, RC: &AArch64::FPR64RegClass, Op0, Op1);
13841 }
13842 return 0;
13843}
13844
13845unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13846 if (RetVT.SimpleTy != MVT::v8i16)
13847 return 0;
13848 if ((Subtarget->isNeonAvailable())) {
13849 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv8i16, RC: &AArch64::FPR128RegClass, Op0, Op1);
13850 }
13851 return 0;
13852}
13853
13854unsigned fastEmit_ISD_USUBSAT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13855 if (RetVT.SimpleTy != MVT::v2i32)
13856 return 0;
13857 if ((Subtarget->isNeonAvailable())) {
13858 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i32, RC: &AArch64::FPR64RegClass, Op0, Op1);
13859 }
13860 return 0;
13861}
13862
13863unsigned fastEmit_ISD_USUBSAT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13864 if (RetVT.SimpleTy != MVT::v4i32)
13865 return 0;
13866 if ((Subtarget->isNeonAvailable())) {
13867 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv4i32, RC: &AArch64::FPR128RegClass, Op0, Op1);
13868 }
13869 return 0;
13870}
13871
13872unsigned fastEmit_ISD_USUBSAT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13873 if (RetVT.SimpleTy != MVT::v2i64)
13874 return 0;
13875 if ((Subtarget->isNeonAvailable())) {
13876 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUBv2i64, RC: &AArch64::FPR128RegClass, Op0, Op1);
13877 }
13878 return 0;
13879}
13880
13881unsigned fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13882 if (RetVT.SimpleTy != MVT::nxv16i8)
13883 return 0;
13884 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13885 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_B, RC: &AArch64::ZPRRegClass, Op0, Op1);
13886 }
13887 return 0;
13888}
13889
13890unsigned fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13891 if (RetVT.SimpleTy != MVT::nxv8i16)
13892 return 0;
13893 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13894 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_H, RC: &AArch64::ZPRRegClass, Op0, Op1);
13895 }
13896 return 0;
13897}
13898
13899unsigned fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13900 if (RetVT.SimpleTy != MVT::nxv4i32)
13901 return 0;
13902 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13903 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_S, RC: &AArch64::ZPRRegClass, Op0, Op1);
13904 }
13905 return 0;
13906}
13907
13908unsigned fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13909 if (RetVT.SimpleTy != MVT::nxv2i64)
13910 return 0;
13911 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
13912 return fastEmitInst_rr(MachineInstOpcode: AArch64::UQSUB_ZZZ_D, RC: &AArch64::ZPRRegClass, Op0, Op1);
13913 }
13914 return 0;
13915}
13916
13917unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
13918 switch (VT.SimpleTy) {
13919 case MVT::v8i8: return fastEmit_ISD_USUBSAT_MVT_v8i8_rr(RetVT, Op0, Op1);
13920 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op1);
13921 case MVT::v4i16: return fastEmit_ISD_USUBSAT_MVT_v4i16_rr(RetVT, Op0, Op1);
13922 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op1);
13923 case MVT::v2i32: return fastEmit_ISD_USUBSAT_MVT_v2i32_rr(RetVT, Op0, Op1);
13924 case MVT::v4i32: return fastEmit_ISD_USUBSAT_MVT_v4i32_rr(RetVT, Op0, Op1);
13925 case MVT::v2i64: return fastEmit_ISD_USUBSAT_MVT_v2i64_rr(RetVT, Op0, Op1);
13926 case MVT::nxv16i8: return fastEmit_ISD_USUBSAT_MVT_nxv16i8_rr(RetVT, Op0, Op1);
13927 case MVT::nxv8i16: return fastEmit_ISD_USUBSAT_MVT_nxv8i16_rr(RetVT, Op0, Op1);
13928 case MVT::nxv4i32: return fastEmit_ISD_USUBSAT_MVT_nxv4i32_rr(RetVT, Op0, Op1);
13929 case MVT::nxv2i64: return fastEmit_ISD_USUBSAT_MVT_nxv2i64_rr(RetVT, Op0, Op1);
13930 default: return 0;
13931 }
13932}
13933
13934// FastEmit functions for ISD::XOR.
13935
13936unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13937 if (RetVT.SimpleTy != MVT::i32)
13938 return 0;
13939 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORWrr, RC: &AArch64::GPR32RegClass, Op0, Op1);
13940}
13941
13942unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13943 if (RetVT.SimpleTy != MVT::i64)
13944 return 0;
13945 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORXrr, RC: &AArch64::GPR64RegClass, Op0, Op1);
13946}
13947
13948unsigned fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13949 if (RetVT.SimpleTy != MVT::v8i8)
13950 return 0;
13951 if ((Subtarget->isNeonAvailable())) {
13952 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13953 }
13954 return 0;
13955}
13956
13957unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13958 if (RetVT.SimpleTy != MVT::v16i8)
13959 return 0;
13960 if ((Subtarget->isNeonAvailable())) {
13961 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13962 }
13963 return 0;
13964}
13965
13966unsigned fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13967 if (RetVT.SimpleTy != MVT::v4i16)
13968 return 0;
13969 if ((Subtarget->isNeonAvailable())) {
13970 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13971 }
13972 return 0;
13973}
13974
13975unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13976 if (RetVT.SimpleTy != MVT::v8i16)
13977 return 0;
13978 if ((Subtarget->isNeonAvailable())) {
13979 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13980 }
13981 return 0;
13982}
13983
13984unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13985 if (RetVT.SimpleTy != MVT::v2i32)
13986 return 0;
13987 if ((Subtarget->isNeonAvailable())) {
13988 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
13989 }
13990 return 0;
13991}
13992
13993unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
13994 if (RetVT.SimpleTy != MVT::v4i32)
13995 return 0;
13996 if ((Subtarget->isNeonAvailable())) {
13997 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
13998 }
13999 return 0;
14000}
14001
14002unsigned fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14003 if (RetVT.SimpleTy != MVT::v1i64)
14004 return 0;
14005 if ((Subtarget->isNeonAvailable())) {
14006 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv8i8, RC: &AArch64::FPR64RegClass, Op0, Op1);
14007 }
14008 return 0;
14009}
14010
14011unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14012 if (RetVT.SimpleTy != MVT::v2i64)
14013 return 0;
14014 if ((Subtarget->isNeonAvailable())) {
14015 return fastEmitInst_rr(MachineInstOpcode: AArch64::EORv16i8, RC: &AArch64::FPR128RegClass, Op0, Op1);
14016 }
14017 return 0;
14018}
14019
14020unsigned fastEmit_ISD_XOR_MVT_nxv16i8_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14021 if (RetVT.SimpleTy != MVT::nxv16i8)
14022 return 0;
14023 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
14024 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14025 }
14026 return 0;
14027}
14028
14029unsigned fastEmit_ISD_XOR_MVT_nxv8i16_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14030 if (RetVT.SimpleTy != MVT::nxv8i16)
14031 return 0;
14032 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
14033 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14034 }
14035 return 0;
14036}
14037
14038unsigned fastEmit_ISD_XOR_MVT_nxv4i32_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14039 if (RetVT.SimpleTy != MVT::nxv4i32)
14040 return 0;
14041 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
14042 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14043 }
14044 return 0;
14045}
14046
14047unsigned fastEmit_ISD_XOR_MVT_nxv2i64_rr(MVT RetVT, unsigned Op0, unsigned Op1) {
14048 if (RetVT.SimpleTy != MVT::nxv2i64)
14049 return 0;
14050 if ((Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME()))) {
14051 return fastEmitInst_rr(MachineInstOpcode: AArch64::EOR_ZZZ, RC: &AArch64::ZPRRegClass, Op0, Op1);
14052 }
14053 return 0;
14054}
14055
14056unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, unsigned Op1) {
14057 switch (VT.SimpleTy) {
14058 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op1);
14059 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op1);
14060 case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op1);
14061 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op1);
14062 case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op1);
14063 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op1);
14064 case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op1);
14065 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op1);
14066 case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op1);
14067 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op1);
14068 case MVT::nxv16i8: return fastEmit_ISD_XOR_MVT_nxv16i8_rr(RetVT, Op0, Op1);
14069 case MVT::nxv8i16: return fastEmit_ISD_XOR_MVT_nxv8i16_rr(RetVT, Op0, Op1);
14070 case MVT::nxv4i32: return fastEmit_ISD_XOR_MVT_nxv4i32_rr(RetVT, Op0, Op1);
14071 case MVT::nxv2i64: return fastEmit_ISD_XOR_MVT_nxv2i64_rr(RetVT, Op0, Op1);
14072 default: return 0;
14073 }
14074}
14075
14076// Top-level FastEmit function.
14077
14078unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, unsigned Op1) override {
14079 switch (Opcode) {
14080 case AArch64ISD::ADDP: return fastEmit_AArch64ISD_ADDP_rr(VT, RetVT, Op0, Op1);
14081 case AArch64ISD::BIC: return fastEmit_AArch64ISD_BIC_rr(VT, RetVT, Op0, Op1);
14082 case AArch64ISD::CMEQ: return fastEmit_AArch64ISD_CMEQ_rr(VT, RetVT, Op0, Op1);
14083 case AArch64ISD::CMGE: return fastEmit_AArch64ISD_CMGE_rr(VT, RetVT, Op0, Op1);
14084 case AArch64ISD::CMGT: return fastEmit_AArch64ISD_CMGT_rr(VT, RetVT, Op0, Op1);
14085 case AArch64ISD::CMHI: return fastEmit_AArch64ISD_CMHI_rr(VT, RetVT, Op0, Op1);
14086 case AArch64ISD::CMHS: return fastEmit_AArch64ISD_CMHS_rr(VT, RetVT, Op0, Op1);
14087 case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op1);
14088 case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op1);
14089 case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op1);
14090 case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op1);
14091 case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op1);
14092 case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op1);
14093 case AArch64ISD::PMULL: return fastEmit_AArch64ISD_PMULL_rr(VT, RetVT, Op0, Op1);
14094 case AArch64ISD::PTEST: return fastEmit_AArch64ISD_PTEST_rr(VT, RetVT, Op0, Op1);
14095 case AArch64ISD::PTEST_ANY: return fastEmit_AArch64ISD_PTEST_ANY_rr(VT, RetVT, Op0, Op1);
14096 case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op1);
14097 case AArch64ISD::STRICT_FCMP: return fastEmit_AArch64ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op1);
14098 case AArch64ISD::STRICT_FCMPE: return fastEmit_AArch64ISD_STRICT_FCMPE_rr(VT, RetVT, Op0, Op1);
14099 case AArch64ISD::TBL: return fastEmit_AArch64ISD_TBL_rr(VT, RetVT, Op0, Op1);
14100 case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op1);
14101 case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op1);
14102 case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op1);
14103 case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op1);
14104 case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op1);
14105 case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op1);
14106 case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op1);
14107 case ISD::ABDS: return fastEmit_ISD_ABDS_rr(VT, RetVT, Op0, Op1);
14108 case ISD::ABDU: return fastEmit_ISD_ABDU_rr(VT, RetVT, Op0, Op1);
14109 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op1);
14110 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op1);
14111 case ISD::AVGCEILS: return fastEmit_ISD_AVGCEILS_rr(VT, RetVT, Op0, Op1);
14112 case ISD::AVGCEILU: return fastEmit_ISD_AVGCEILU_rr(VT, RetVT, Op0, Op1);
14113 case ISD::AVGFLOORS: return fastEmit_ISD_AVGFLOORS_rr(VT, RetVT, Op0, Op1);
14114 case ISD::AVGFLOORU: return fastEmit_ISD_AVGFLOORU_rr(VT, RetVT, Op0, Op1);
14115 case ISD::CONCAT_VECTORS: return fastEmit_ISD_CONCAT_VECTORS_rr(VT, RetVT, Op0, Op1);
14116 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op1);
14117 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op1);
14118 case ISD::FMAXIMUM: return fastEmit_ISD_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14119 case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14120 case ISD::FMINIMUM: return fastEmit_ISD_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14121 case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op1);
14122 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op1);
14123 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op1);
14124 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op1);
14125 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op1);
14126 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op1);
14127 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op1);
14128 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op1);
14129 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op1);
14130 case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op1);
14131 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op1);
14132 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op1);
14133 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op1);
14134 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op1);
14135 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op1);
14136 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op1);
14137 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op1);
14138 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op1);
14139 case ISD::STRICT_FMAXIMUM: return fastEmit_ISD_STRICT_FMAXIMUM_rr(VT, RetVT, Op0, Op1);
14140 case ISD::STRICT_FMAXNUM: return fastEmit_ISD_STRICT_FMAXNUM_rr(VT, RetVT, Op0, Op1);
14141 case ISD::STRICT_FMINIMUM: return fastEmit_ISD_STRICT_FMINIMUM_rr(VT, RetVT, Op0, Op1);
14142 case ISD::STRICT_FMINNUM: return fastEmit_ISD_STRICT_FMINNUM_rr(VT, RetVT, Op0, Op1);
14143 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op1);
14144 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op1);
14145 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op1);
14146 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op1);
14147 case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op1);
14148 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op1);
14149 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op1);
14150 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op1);
14151 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op1);
14152 default: return 0;
14153 }
14154}
14155
14156// FastEmit functions for AArch64ISD::DUPLANE64.
14157
14158unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) {
14159 if (RetVT.SimpleTy != MVT::v2i64)
14160 return 0;
14161 if ((Subtarget->isNeonAvailable())) {
14162 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14163 }
14164 return 0;
14165}
14166
14167unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) {
14168 if (RetVT.SimpleTy != MVT::v2f64)
14169 return 0;
14170 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i64lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14171}
14172
14173unsigned fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14174 switch (VT.SimpleTy) {
14175 case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14176 case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14177 default: return 0;
14178 }
14179}
14180
14181// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14182
14183unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) {
14184 if (RetVT.SimpleTy != MVT::i64)
14185 return 0;
14186 if ((Subtarget->isNeonAvailable())) {
14187 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14188 }
14189 return 0;
14190}
14191
14192unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, uint64_t imm1) {
14193 if (RetVT.SimpleTy != MVT::f64)
14194 return 0;
14195 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi64, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14196}
14197
14198unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14199 switch (VT.SimpleTy) {
14200 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14201 case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, imm1);
14202 default: return 0;
14203 }
14204}
14205
14206// Top-level FastEmit function.
14207
14208unsigned fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14209 switch (Opcode) {
14210 case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14211 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, imm1);
14212 default: return 0;
14213 }
14214}
14215
14216// FastEmit functions for AArch64ISD::DUPLANE32.
14217
14218unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) {
14219 if ((Subtarget->isNeonAvailable())) {
14220 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14221 }
14222 return 0;
14223}
14224
14225unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) {
14226 if ((Subtarget->isNeonAvailable())) {
14227 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14228 }
14229 return 0;
14230}
14231
14232unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) {
14233switch (RetVT.SimpleTy) {
14234 case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, imm1);
14235 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, imm1);
14236 default: return 0;
14237}
14238}
14239
14240unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) {
14241 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv2i32lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14242}
14243
14244unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(unsigned Op0, uint64_t imm1) {
14245 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i32lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14246}
14247
14248unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) {
14249switch (RetVT.SimpleTy) {
14250 case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, imm1);
14251 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, imm1);
14252 default: return 0;
14253}
14254}
14255
14256unsigned fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14257 switch (VT.SimpleTy) {
14258 case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14259 case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14260 default: return 0;
14261 }
14262}
14263
14264// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14265
14266unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) {
14267 if (RetVT.SimpleTy != MVT::i32)
14268 return 0;
14269 if ((Subtarget->isNeonAvailable())) {
14270 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14271 }
14272 return 0;
14273}
14274
14275unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, uint64_t imm1) {
14276 if (RetVT.SimpleTy != MVT::f32)
14277 return 0;
14278 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi32, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14279}
14280
14281unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14282 switch (VT.SimpleTy) {
14283 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14284 case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, imm1);
14285 default: return 0;
14286 }
14287}
14288
14289// Top-level FastEmit function.
14290
14291unsigned fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14292 switch (Opcode) {
14293 case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14294 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, imm1);
14295 default: return 0;
14296 }
14297}
14298
14299// FastEmit functions for AArch64ISD::DUPLANE16.
14300
14301unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) {
14302 if ((Subtarget->isNeonAvailable())) {
14303 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14304 }
14305 return 0;
14306}
14307
14308unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) {
14309 if ((Subtarget->isNeonAvailable())) {
14310 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14311 }
14312 return 0;
14313}
14314
14315unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) {
14316switch (RetVT.SimpleTy) {
14317 case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, imm1);
14318 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, imm1);
14319 default: return 0;
14320}
14321}
14322
14323unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) {
14324 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14325}
14326
14327unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) {
14328 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14329}
14330
14331unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) {
14332switch (RetVT.SimpleTy) {
14333 case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, imm1);
14334 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, imm1);
14335 default: return 0;
14336}
14337}
14338
14339unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) {
14340 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv4i16lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14341}
14342
14343unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(unsigned Op0, uint64_t imm1) {
14344 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i16lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14345}
14346
14347unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) {
14348switch (RetVT.SimpleTy) {
14349 case MVT::v4bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v4bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14350 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_MVT_v8bf16_ri_Predicate_VectorIndexH(Op0, imm1);
14351 default: return 0;
14352}
14353}
14354
14355unsigned fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14356 switch (VT.SimpleTy) {
14357 case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14358 case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14359 case MVT::v8bf16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14360 default: return 0;
14361 }
14362}
14363
14364// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14365
14366unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) {
14367 if (RetVT.SimpleTy != MVT::i32)
14368 return 0;
14369 if ((Subtarget->isNeonAvailable())) {
14370 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14371 }
14372 return 0;
14373}
14374
14375unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) {
14376 if (RetVT.SimpleTy != MVT::f16)
14377 return 0;
14378 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14379}
14380
14381unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, uint64_t imm1) {
14382 if (RetVT.SimpleTy != MVT::bf16)
14383 return 0;
14384 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPi16, RC: &AArch64::FPR16RegClass, Op0, Imm: imm1);
14385}
14386
14387unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14388 switch (VT.SimpleTy) {
14389 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14390 case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14391 case MVT::v8bf16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8bf16_ri_Predicate_VectorIndexH(RetVT, Op0, imm1);
14392 default: return 0;
14393 }
14394}
14395
14396// Top-level FastEmit function.
14397
14398unsigned fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14399 switch (Opcode) {
14400 case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14401 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, imm1);
14402 default: return 0;
14403 }
14404}
14405
14406// FastEmit functions for AArch64ISD::DUPLANE8.
14407
14408unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(unsigned Op0, uint64_t imm1) {
14409 if ((Subtarget->isNeonAvailable())) {
14410 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv8i8lane, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14411 }
14412 return 0;
14413}
14414
14415unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(unsigned Op0, uint64_t imm1) {
14416 if ((Subtarget->isNeonAvailable())) {
14417 return fastEmitInst_ri(MachineInstOpcode: AArch64::DUPv16i8lane, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14418 }
14419 return 0;
14420}
14421
14422unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, uint64_t imm1) {
14423switch (RetVT.SimpleTy) {
14424 case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, imm1);
14425 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, imm1);
14426 default: return 0;
14427}
14428}
14429
14430unsigned fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14431 switch (VT.SimpleTy) {
14432 case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14433 default: return 0;
14434 }
14435}
14436
14437// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14438
14439unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, uint64_t imm1) {
14440 if (RetVT.SimpleTy != MVT::i32)
14441 return 0;
14442 if ((Subtarget->isNeonAvailable())) {
14443 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14444 }
14445 return 0;
14446}
14447
14448unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14449 switch (VT.SimpleTy) {
14450 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, imm1);
14451 default: return 0;
14452 }
14453}
14454
14455// Top-level FastEmit function.
14456
14457unsigned fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14458 switch (Opcode) {
14459 case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14460 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, imm1);
14461 default: return 0;
14462 }
14463}
14464
14465// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
14466
14467unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) {
14468 if (RetVT.SimpleTy != MVT::i32)
14469 return 0;
14470 if ((Subtarget->hasNEON())) {
14471 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi8_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14472 }
14473 return 0;
14474}
14475
14476unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) {
14477 if (RetVT.SimpleTy != MVT::i32)
14478 return 0;
14479 if ((Subtarget->hasNEON())) {
14480 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi16_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14481 }
14482 return 0;
14483}
14484
14485unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) {
14486 if (RetVT.SimpleTy != MVT::i32)
14487 return 0;
14488 if ((Subtarget->hasNEON())) {
14489 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi32_idx0, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
14490 }
14491 return 0;
14492}
14493
14494unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(MVT RetVT, unsigned Op0, uint64_t imm1) {
14495 if (RetVT.SimpleTy != MVT::i64)
14496 return 0;
14497 if ((Subtarget->hasNEON())) {
14498 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMOVvi64_idx0, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
14499 }
14500 return 0;
14501}
14502
14503unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14504 switch (VT.SimpleTy) {
14505 case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14506 case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14507 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14508 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndex0(RetVT, Op0, imm1);
14509 default: return 0;
14510 }
14511}
14512
14513// Top-level FastEmit function.
14514
14515unsigned fastEmit_ri_Predicate_VectorIndex0(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14516 switch (Opcode) {
14517 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndex0(VT, RetVT, Op0, imm1);
14518 default: return 0;
14519 }
14520}
14521
14522// FastEmit functions for AArch64ISD::SQSHLU_I.
14523
14524unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14525 if (RetVT.SimpleTy != MVT::i64)
14526 return 0;
14527 if ((Subtarget->isNeonAvailable())) {
14528 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14529 }
14530 return 0;
14531}
14532
14533unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14534 if (RetVT.SimpleTy != MVT::v1i64)
14535 return 0;
14536 if ((Subtarget->isNeonAvailable())) {
14537 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14538 }
14539 return 0;
14540}
14541
14542unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14543 if (RetVT.SimpleTy != MVT::v2i64)
14544 return 0;
14545 if ((Subtarget->isNeonAvailable())) {
14546 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14547 }
14548 return 0;
14549}
14550
14551unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14552 switch (VT.SimpleTy) {
14553 case MVT::i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14554 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14555 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14556 default: return 0;
14557 }
14558}
14559
14560// FastEmit functions for AArch64ISD::SQSHL_I.
14561
14562unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14563 if (RetVT.SimpleTy != MVT::i64)
14564 return 0;
14565 if ((Subtarget->isNeonAvailable())) {
14566 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14567 }
14568 return 0;
14569}
14570
14571unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14572 if (RetVT.SimpleTy != MVT::v1i64)
14573 return 0;
14574 if ((Subtarget->isNeonAvailable())) {
14575 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14576 }
14577 return 0;
14578}
14579
14580unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14581 if (RetVT.SimpleTy != MVT::v2i64)
14582 return 0;
14583 if ((Subtarget->isNeonAvailable())) {
14584 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14585 }
14586 return 0;
14587}
14588
14589unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14590 switch (VT.SimpleTy) {
14591 case MVT::i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14592 case MVT::v1i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14593 case MVT::v2i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14594 default: return 0;
14595 }
14596}
14597
14598// FastEmit functions for AArch64ISD::UQSHL_I.
14599
14600unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14601 if (RetVT.SimpleTy != MVT::i64)
14602 return 0;
14603 if ((Subtarget->isNeonAvailable())) {
14604 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14605 }
14606 return 0;
14607}
14608
14609unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14610 if (RetVT.SimpleTy != MVT::v1i64)
14611 return 0;
14612 if ((Subtarget->isNeonAvailable())) {
14613 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14614 }
14615 return 0;
14616}
14617
14618unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14619 if (RetVT.SimpleTy != MVT::v2i64)
14620 return 0;
14621 if ((Subtarget->isNeonAvailable())) {
14622 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14623 }
14624 return 0;
14625}
14626
14627unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14628 switch (VT.SimpleTy) {
14629 case MVT::i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14630 case MVT::v1i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14631 case MVT::v2i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14632 default: return 0;
14633 }
14634}
14635
14636// FastEmit functions for AArch64ISD::VSHL.
14637
14638unsigned fastEmit_AArch64ISD_VSHL_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14639 if (RetVT.SimpleTy != MVT::i64)
14640 return 0;
14641 if ((Subtarget->isNeonAvailable())) {
14642 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14643 }
14644 return 0;
14645}
14646
14647unsigned fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14648 if (RetVT.SimpleTy != MVT::v1i64)
14649 return 0;
14650 if ((Subtarget->isNeonAvailable())) {
14651 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14652 }
14653 return 0;
14654}
14655
14656unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14657 if (RetVT.SimpleTy != MVT::v2i64)
14658 return 0;
14659 if ((Subtarget->isNeonAvailable())) {
14660 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14661 }
14662 return 0;
14663}
14664
14665unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14666 switch (VT.SimpleTy) {
14667 case MVT::i64: return fastEmit_AArch64ISD_VSHL_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14668 case MVT::v1i64: return fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14669 case MVT::v2i64: return fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, imm1);
14670 default: return 0;
14671 }
14672}
14673
14674// Top-level FastEmit function.
14675
14676unsigned fastEmit_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14677 switch (Opcode) {
14678 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14679 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14680 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14681 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(VT, RetVT, Op0, imm1);
14682 default: return 0;
14683 }
14684}
14685
14686// FastEmit functions for AArch64ISD::SQSHLU_I.
14687
14688unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14689 if (RetVT.SimpleTy != MVT::i32)
14690 return 0;
14691 if ((Subtarget->isNeonAvailable())) {
14692 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUs, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14693 }
14694 return 0;
14695}
14696
14697unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14698 if (RetVT.SimpleTy != MVT::v2i32)
14699 return 0;
14700 if ((Subtarget->isNeonAvailable())) {
14701 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14702 }
14703 return 0;
14704}
14705
14706unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14707 if (RetVT.SimpleTy != MVT::v4i32)
14708 return 0;
14709 if ((Subtarget->isNeonAvailable())) {
14710 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14711 }
14712 return 0;
14713}
14714
14715unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14716 switch (VT.SimpleTy) {
14717 case MVT::i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14718 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14719 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14720 default: return 0;
14721 }
14722}
14723
14724// FastEmit functions for AArch64ISD::SQSHL_I.
14725
14726unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14727 if (RetVT.SimpleTy != MVT::i32)
14728 return 0;
14729 if ((Subtarget->isNeonAvailable())) {
14730 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLs, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14731 }
14732 return 0;
14733}
14734
14735unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14736 if (RetVT.SimpleTy != MVT::v2i32)
14737 return 0;
14738 if ((Subtarget->isNeonAvailable())) {
14739 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14740 }
14741 return 0;
14742}
14743
14744unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14745 if (RetVT.SimpleTy != MVT::v4i32)
14746 return 0;
14747 if ((Subtarget->isNeonAvailable())) {
14748 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14749 }
14750 return 0;
14751}
14752
14753unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14754 switch (VT.SimpleTy) {
14755 case MVT::i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14756 case MVT::v2i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14757 case MVT::v4i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14758 default: return 0;
14759 }
14760}
14761
14762// FastEmit functions for AArch64ISD::UQSHL_I.
14763
14764unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14765 if (RetVT.SimpleTy != MVT::i32)
14766 return 0;
14767 if ((Subtarget->isNeonAvailable())) {
14768 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLs, RC: &AArch64::FPR32RegClass, Op0, Imm: imm1);
14769 }
14770 return 0;
14771}
14772
14773unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14774 if (RetVT.SimpleTy != MVT::v2i32)
14775 return 0;
14776 if ((Subtarget->isNeonAvailable())) {
14777 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14778 }
14779 return 0;
14780}
14781
14782unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14783 if (RetVT.SimpleTy != MVT::v4i32)
14784 return 0;
14785 if ((Subtarget->isNeonAvailable())) {
14786 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14787 }
14788 return 0;
14789}
14790
14791unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14792 switch (VT.SimpleTy) {
14793 case MVT::i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14794 case MVT::v2i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14795 case MVT::v4i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14796 default: return 0;
14797 }
14798}
14799
14800// FastEmit functions for AArch64ISD::VSHL.
14801
14802unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14803 if (RetVT.SimpleTy != MVT::v2i32)
14804 return 0;
14805 if ((Subtarget->isNeonAvailable())) {
14806 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14807 }
14808 return 0;
14809}
14810
14811unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, uint64_t imm1) {
14812 if (RetVT.SimpleTy != MVT::v4i32)
14813 return 0;
14814 if ((Subtarget->isNeonAvailable())) {
14815 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14816 }
14817 return 0;
14818}
14819
14820unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14821 switch (VT.SimpleTy) {
14822 case MVT::v2i32: return fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14823 case MVT::v4i32: return fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, imm1);
14824 default: return 0;
14825 }
14826}
14827
14828// Top-level FastEmit function.
14829
14830unsigned fastEmit_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14831 switch (Opcode) {
14832 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14833 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14834 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14835 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(VT, RetVT, Op0, imm1);
14836 default: return 0;
14837 }
14838}
14839
14840// FastEmit functions for AArch64ISD::SRSHR_I.
14841
14842unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14843 if (RetVT.SimpleTy != MVT::i64)
14844 return 0;
14845 if ((Subtarget->isNeonAvailable())) {
14846 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14847 }
14848 return 0;
14849}
14850
14851unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14852 if (RetVT.SimpleTy != MVT::v1i64)
14853 return 0;
14854 if ((Subtarget->isNeonAvailable())) {
14855 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14856 }
14857 return 0;
14858}
14859
14860unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14861 if (RetVT.SimpleTy != MVT::v2i64)
14862 return 0;
14863 if ((Subtarget->isNeonAvailable())) {
14864 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14865 }
14866 return 0;
14867}
14868
14869unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14870 switch (VT.SimpleTy) {
14871 case MVT::i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14872 case MVT::v1i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14873 case MVT::v2i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14874 default: return 0;
14875 }
14876}
14877
14878// FastEmit functions for AArch64ISD::URSHR_I.
14879
14880unsigned fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14881 if (RetVT.SimpleTy != MVT::i64)
14882 return 0;
14883 if ((Subtarget->isNeonAvailable())) {
14884 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14885 }
14886 return 0;
14887}
14888
14889unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14890 if (RetVT.SimpleTy != MVT::v1i64)
14891 return 0;
14892 if ((Subtarget->isNeonAvailable())) {
14893 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14894 }
14895 return 0;
14896}
14897
14898unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14899 if (RetVT.SimpleTy != MVT::v2i64)
14900 return 0;
14901 if ((Subtarget->isNeonAvailable())) {
14902 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14903 }
14904 return 0;
14905}
14906
14907unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14908 switch (VT.SimpleTy) {
14909 case MVT::i64: return fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14910 case MVT::v1i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14911 case MVT::v2i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14912 default: return 0;
14913 }
14914}
14915
14916// FastEmit functions for AArch64ISD::VASHR.
14917
14918unsigned fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14919 if (RetVT.SimpleTy != MVT::i64)
14920 return 0;
14921 if ((Subtarget->isNeonAvailable())) {
14922 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14923 }
14924 return 0;
14925}
14926
14927unsigned fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14928 if (RetVT.SimpleTy != MVT::v1i64)
14929 return 0;
14930 if ((Subtarget->isNeonAvailable())) {
14931 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14932 }
14933 return 0;
14934}
14935
14936unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14937 if (RetVT.SimpleTy != MVT::v2i64)
14938 return 0;
14939 if ((Subtarget->isNeonAvailable())) {
14940 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14941 }
14942 return 0;
14943}
14944
14945unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14946 switch (VT.SimpleTy) {
14947 case MVT::i64: return fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14948 case MVT::v1i64: return fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14949 case MVT::v2i64: return fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14950 default: return 0;
14951 }
14952}
14953
14954// FastEmit functions for AArch64ISD::VLSHR.
14955
14956unsigned fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14957 if (RetVT.SimpleTy != MVT::i64)
14958 return 0;
14959 if ((Subtarget->isNeonAvailable())) {
14960 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14961 }
14962 return 0;
14963}
14964
14965unsigned fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14966 if (RetVT.SimpleTy != MVT::v1i64)
14967 return 0;
14968 if ((Subtarget->isNeonAvailable())) {
14969 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRd, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
14970 }
14971 return 0;
14972}
14973
14974unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, uint64_t imm1) {
14975 if (RetVT.SimpleTy != MVT::v2i64)
14976 return 0;
14977 if ((Subtarget->isNeonAvailable())) {
14978 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv2i64_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
14979 }
14980 return 0;
14981}
14982
14983unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
14984 switch (VT.SimpleTy) {
14985 case MVT::i64: return fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14986 case MVT::v1i64: return fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14987 case MVT::v2i64: return fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, imm1);
14988 default: return 0;
14989 }
14990}
14991
14992// Top-level FastEmit function.
14993
14994unsigned fastEmit_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
14995 switch (Opcode) {
14996 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14997 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14998 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
14999 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, imm1);
15000 default: return 0;
15001 }
15002}
15003
15004// FastEmit functions for AArch64ISD::SQSHLU_I.
15005
15006unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15007 if (RetVT.SimpleTy != MVT::v8i8)
15008 return 0;
15009 if ((Subtarget->isNeonAvailable())) {
15010 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15011 }
15012 return 0;
15013}
15014
15015unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15016 if (RetVT.SimpleTy != MVT::v16i8)
15017 return 0;
15018 if ((Subtarget->isNeonAvailable())) {
15019 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15020 }
15021 return 0;
15022}
15023
15024unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15025 switch (VT.SimpleTy) {
15026 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15027 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15028 default: return 0;
15029 }
15030}
15031
15032// FastEmit functions for AArch64ISD::SQSHL_I.
15033
15034unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15035 if (RetVT.SimpleTy != MVT::v8i8)
15036 return 0;
15037 if ((Subtarget->isNeonAvailable())) {
15038 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15039 }
15040 return 0;
15041}
15042
15043unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15044 if (RetVT.SimpleTy != MVT::v16i8)
15045 return 0;
15046 if ((Subtarget->isNeonAvailable())) {
15047 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15048 }
15049 return 0;
15050}
15051
15052unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15053 switch (VT.SimpleTy) {
15054 case MVT::v8i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15055 case MVT::v16i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15056 default: return 0;
15057 }
15058}
15059
15060// FastEmit functions for AArch64ISD::UQSHL_I.
15061
15062unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15063 if (RetVT.SimpleTy != MVT::v8i8)
15064 return 0;
15065 if ((Subtarget->isNeonAvailable())) {
15066 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15067 }
15068 return 0;
15069}
15070
15071unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15072 if (RetVT.SimpleTy != MVT::v16i8)
15073 return 0;
15074 if ((Subtarget->isNeonAvailable())) {
15075 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15076 }
15077 return 0;
15078}
15079
15080unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15081 switch (VT.SimpleTy) {
15082 case MVT::v8i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15083 case MVT::v16i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15084 default: return 0;
15085 }
15086}
15087
15088// FastEmit functions for AArch64ISD::VSHL.
15089
15090unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15091 if (RetVT.SimpleTy != MVT::v8i8)
15092 return 0;
15093 if ((Subtarget->isNeonAvailable())) {
15094 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15095 }
15096 return 0;
15097}
15098
15099unsigned fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15100 if (RetVT.SimpleTy != MVT::v16i8)
15101 return 0;
15102 if ((Subtarget->isNeonAvailable())) {
15103 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15104 }
15105 return 0;
15106}
15107
15108unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15109 switch (VT.SimpleTy) {
15110 case MVT::v8i8: return fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15111 case MVT::v16i8: return fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, imm1);
15112 default: return 0;
15113 }
15114}
15115
15116// Top-level FastEmit function.
15117
15118unsigned fastEmit_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15119 switch (Opcode) {
15120 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
15121 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
15122 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
15123 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(VT, RetVT, Op0, imm1);
15124 default: return 0;
15125 }
15126}
15127
15128// FastEmit functions for AArch64ISD::SQSHLU_I.
15129
15130unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15131 if (RetVT.SimpleTy != MVT::v4i16)
15132 return 0;
15133 if ((Subtarget->isNeonAvailable())) {
15134 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15135 }
15136 return 0;
15137}
15138
15139unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15140 if (RetVT.SimpleTy != MVT::v8i16)
15141 return 0;
15142 if ((Subtarget->isNeonAvailable())) {
15143 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLUv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15144 }
15145 return 0;
15146}
15147
15148unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15149 switch (VT.SimpleTy) {
15150 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15151 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15152 default: return 0;
15153 }
15154}
15155
15156// FastEmit functions for AArch64ISD::SQSHL_I.
15157
15158unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15159 if (RetVT.SimpleTy != MVT::v4i16)
15160 return 0;
15161 if ((Subtarget->isNeonAvailable())) {
15162 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15163 }
15164 return 0;
15165}
15166
15167unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15168 if (RetVT.SimpleTy != MVT::v8i16)
15169 return 0;
15170 if ((Subtarget->isNeonAvailable())) {
15171 return fastEmitInst_ri(MachineInstOpcode: AArch64::SQSHLv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15172 }
15173 return 0;
15174}
15175
15176unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15177 switch (VT.SimpleTy) {
15178 case MVT::v4i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15179 case MVT::v8i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15180 default: return 0;
15181 }
15182}
15183
15184// FastEmit functions for AArch64ISD::UQSHL_I.
15185
15186unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15187 if (RetVT.SimpleTy != MVT::v4i16)
15188 return 0;
15189 if ((Subtarget->isNeonAvailable())) {
15190 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15191 }
15192 return 0;
15193}
15194
15195unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15196 if (RetVT.SimpleTy != MVT::v8i16)
15197 return 0;
15198 if ((Subtarget->isNeonAvailable())) {
15199 return fastEmitInst_ri(MachineInstOpcode: AArch64::UQSHLv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15200 }
15201 return 0;
15202}
15203
15204unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15205 switch (VT.SimpleTy) {
15206 case MVT::v4i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15207 case MVT::v8i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15208 default: return 0;
15209 }
15210}
15211
15212// FastEmit functions for AArch64ISD::VSHL.
15213
15214unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15215 if (RetVT.SimpleTy != MVT::v4i16)
15216 return 0;
15217 if ((Subtarget->isNeonAvailable())) {
15218 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15219 }
15220 return 0;
15221}
15222
15223unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15224 if (RetVT.SimpleTy != MVT::v8i16)
15225 return 0;
15226 if ((Subtarget->isNeonAvailable())) {
15227 return fastEmitInst_ri(MachineInstOpcode: AArch64::SHLv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15228 }
15229 return 0;
15230}
15231
15232unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15233 switch (VT.SimpleTy) {
15234 case MVT::v4i16: return fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15235 case MVT::v8i16: return fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, imm1);
15236 default: return 0;
15237 }
15238}
15239
15240// Top-level FastEmit function.
15241
15242unsigned fastEmit_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15243 switch (Opcode) {
15244 case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
15245 case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
15246 case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
15247 case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(VT, RetVT, Op0, imm1);
15248 default: return 0;
15249 }
15250}
15251
15252// FastEmit functions for AArch64ISD::SRSHR_I.
15253
15254unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15255 if (RetVT.SimpleTy != MVT::v8i8)
15256 return 0;
15257 if ((Subtarget->isNeonAvailable())) {
15258 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15259 }
15260 return 0;
15261}
15262
15263unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15264 if (RetVT.SimpleTy != MVT::v16i8)
15265 return 0;
15266 if ((Subtarget->isNeonAvailable())) {
15267 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15268 }
15269 return 0;
15270}
15271
15272unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15273 switch (VT.SimpleTy) {
15274 case MVT::v8i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15275 case MVT::v16i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15276 default: return 0;
15277 }
15278}
15279
15280// FastEmit functions for AArch64ISD::URSHR_I.
15281
15282unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15283 if (RetVT.SimpleTy != MVT::v8i8)
15284 return 0;
15285 if ((Subtarget->isNeonAvailable())) {
15286 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15287 }
15288 return 0;
15289}
15290
15291unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15292 if (RetVT.SimpleTy != MVT::v16i8)
15293 return 0;
15294 if ((Subtarget->isNeonAvailable())) {
15295 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15296 }
15297 return 0;
15298}
15299
15300unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15301 switch (VT.SimpleTy) {
15302 case MVT::v8i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15303 case MVT::v16i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15304 default: return 0;
15305 }
15306}
15307
15308// FastEmit functions for AArch64ISD::VASHR.
15309
15310unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15311 if (RetVT.SimpleTy != MVT::v8i8)
15312 return 0;
15313 if ((Subtarget->isNeonAvailable())) {
15314 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15315 }
15316 return 0;
15317}
15318
15319unsigned fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15320 if (RetVT.SimpleTy != MVT::v16i8)
15321 return 0;
15322 if ((Subtarget->isNeonAvailable())) {
15323 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15324 }
15325 return 0;
15326}
15327
15328unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15329 switch (VT.SimpleTy) {
15330 case MVT::v8i8: return fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15331 case MVT::v16i8: return fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15332 default: return 0;
15333 }
15334}
15335
15336// FastEmit functions for AArch64ISD::VLSHR.
15337
15338unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15339 if (RetVT.SimpleTy != MVT::v8i8)
15340 return 0;
15341 if ((Subtarget->isNeonAvailable())) {
15342 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv8i8_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15343 }
15344 return 0;
15345}
15346
15347unsigned fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, uint64_t imm1) {
15348 if (RetVT.SimpleTy != MVT::v16i8)
15349 return 0;
15350 if ((Subtarget->isNeonAvailable())) {
15351 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv16i8_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15352 }
15353 return 0;
15354}
15355
15356unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15357 switch (VT.SimpleTy) {
15358 case MVT::v8i8: return fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15359 case MVT::v16i8: return fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, imm1);
15360 default: return 0;
15361 }
15362}
15363
15364// Top-level FastEmit function.
15365
15366unsigned fastEmit_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15367 switch (Opcode) {
15368 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15369 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15370 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15371 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, imm1);
15372 default: return 0;
15373 }
15374}
15375
15376// FastEmit functions for AArch64ISD::SRSHR_I.
15377
15378unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15379 if (RetVT.SimpleTy != MVT::v4i16)
15380 return 0;
15381 if ((Subtarget->isNeonAvailable())) {
15382 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15383 }
15384 return 0;
15385}
15386
15387unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15388 if (RetVT.SimpleTy != MVT::v8i16)
15389 return 0;
15390 if ((Subtarget->isNeonAvailable())) {
15391 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15392 }
15393 return 0;
15394}
15395
15396unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15397 switch (VT.SimpleTy) {
15398 case MVT::v4i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15399 case MVT::v8i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15400 default: return 0;
15401 }
15402}
15403
15404// FastEmit functions for AArch64ISD::URSHR_I.
15405
15406unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15407 if (RetVT.SimpleTy != MVT::v4i16)
15408 return 0;
15409 if ((Subtarget->isNeonAvailable())) {
15410 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15411 }
15412 return 0;
15413}
15414
15415unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15416 if (RetVT.SimpleTy != MVT::v8i16)
15417 return 0;
15418 if ((Subtarget->isNeonAvailable())) {
15419 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15420 }
15421 return 0;
15422}
15423
15424unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15425 switch (VT.SimpleTy) {
15426 case MVT::v4i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15427 case MVT::v8i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15428 default: return 0;
15429 }
15430}
15431
15432// FastEmit functions for AArch64ISD::VASHR.
15433
15434unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15435 if (RetVT.SimpleTy != MVT::v4i16)
15436 return 0;
15437 if ((Subtarget->isNeonAvailable())) {
15438 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15439 }
15440 return 0;
15441}
15442
15443unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15444 if (RetVT.SimpleTy != MVT::v8i16)
15445 return 0;
15446 if ((Subtarget->isNeonAvailable())) {
15447 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15448 }
15449 return 0;
15450}
15451
15452unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15453 switch (VT.SimpleTy) {
15454 case MVT::v4i16: return fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15455 case MVT::v8i16: return fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15456 default: return 0;
15457 }
15458}
15459
15460// FastEmit functions for AArch64ISD::VLSHR.
15461
15462unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15463 if (RetVT.SimpleTy != MVT::v4i16)
15464 return 0;
15465 if ((Subtarget->isNeonAvailable())) {
15466 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv4i16_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15467 }
15468 return 0;
15469}
15470
15471unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, uint64_t imm1) {
15472 if (RetVT.SimpleTy != MVT::v8i16)
15473 return 0;
15474 if ((Subtarget->isNeonAvailable())) {
15475 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv8i16_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15476 }
15477 return 0;
15478}
15479
15480unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15481 switch (VT.SimpleTy) {
15482 case MVT::v4i16: return fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15483 case MVT::v8i16: return fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, imm1);
15484 default: return 0;
15485 }
15486}
15487
15488// Top-level FastEmit function.
15489
15490unsigned fastEmit_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15491 switch (Opcode) {
15492 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15493 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15494 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15495 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, imm1);
15496 default: return 0;
15497 }
15498}
15499
15500// FastEmit functions for AArch64ISD::SRSHR_I.
15501
15502unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15503 if (RetVT.SimpleTy != MVT::v2i32)
15504 return 0;
15505 if ((Subtarget->isNeonAvailable())) {
15506 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15507 }
15508 return 0;
15509}
15510
15511unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15512 if (RetVT.SimpleTy != MVT::v4i32)
15513 return 0;
15514 if ((Subtarget->isNeonAvailable())) {
15515 return fastEmitInst_ri(MachineInstOpcode: AArch64::SRSHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15516 }
15517 return 0;
15518}
15519
15520unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15521 switch (VT.SimpleTy) {
15522 case MVT::v2i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15523 case MVT::v4i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15524 default: return 0;
15525 }
15526}
15527
15528// FastEmit functions for AArch64ISD::URSHR_I.
15529
15530unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15531 if (RetVT.SimpleTy != MVT::v2i32)
15532 return 0;
15533 if ((Subtarget->isNeonAvailable())) {
15534 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15535 }
15536 return 0;
15537}
15538
15539unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15540 if (RetVT.SimpleTy != MVT::v4i32)
15541 return 0;
15542 if ((Subtarget->isNeonAvailable())) {
15543 return fastEmitInst_ri(MachineInstOpcode: AArch64::URSHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15544 }
15545 return 0;
15546}
15547
15548unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15549 switch (VT.SimpleTy) {
15550 case MVT::v2i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15551 case MVT::v4i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15552 default: return 0;
15553 }
15554}
15555
15556// FastEmit functions for AArch64ISD::VASHR.
15557
15558unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15559 if (RetVT.SimpleTy != MVT::v2i32)
15560 return 0;
15561 if ((Subtarget->isNeonAvailable())) {
15562 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15563 }
15564 return 0;
15565}
15566
15567unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15568 if (RetVT.SimpleTy != MVT::v4i32)
15569 return 0;
15570 if ((Subtarget->isNeonAvailable())) {
15571 return fastEmitInst_ri(MachineInstOpcode: AArch64::SSHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15572 }
15573 return 0;
15574}
15575
15576unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15577 switch (VT.SimpleTy) {
15578 case MVT::v2i32: return fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15579 case MVT::v4i32: return fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15580 default: return 0;
15581 }
15582}
15583
15584// FastEmit functions for AArch64ISD::VLSHR.
15585
15586unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15587 if (RetVT.SimpleTy != MVT::v2i32)
15588 return 0;
15589 if ((Subtarget->isNeonAvailable())) {
15590 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv2i32_shift, RC: &AArch64::FPR64RegClass, Op0, Imm: imm1);
15591 }
15592 return 0;
15593}
15594
15595unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, uint64_t imm1) {
15596 if (RetVT.SimpleTy != MVT::v4i32)
15597 return 0;
15598 if ((Subtarget->isNeonAvailable())) {
15599 return fastEmitInst_ri(MachineInstOpcode: AArch64::USHRv4i32_shift, RC: &AArch64::FPR128RegClass, Op0, Imm: imm1);
15600 }
15601 return 0;
15602}
15603
15604unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15605 switch (VT.SimpleTy) {
15606 case MVT::v2i32: return fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15607 case MVT::v4i32: return fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, imm1);
15608 default: return 0;
15609 }
15610}
15611
15612// Top-level FastEmit function.
15613
15614unsigned fastEmit_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15615 switch (Opcode) {
15616 case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15617 case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15618 case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15619 case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, imm1);
15620 default: return 0;
15621 }
15622}
15623
15624// FastEmit functions for ISD::SMAX.
15625
15626unsigned fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15627 if (RetVT.SimpleTy != MVT::i32)
15628 return 0;
15629 if ((Subtarget->hasCSSC())) {
15630 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15631 }
15632 return 0;
15633}
15634
15635unsigned fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15636 switch (VT.SimpleTy) {
15637 case MVT::i32: return fastEmit_ISD_SMAX_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
15638 default: return 0;
15639 }
15640}
15641
15642// FastEmit functions for ISD::SMIN.
15643
15644unsigned fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15645 if (RetVT.SimpleTy != MVT::i32)
15646 return 0;
15647 if ((Subtarget->hasCSSC())) {
15648 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15649 }
15650 return 0;
15651}
15652
15653unsigned fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15654 switch (VT.SimpleTy) {
15655 case MVT::i32: return fastEmit_ISD_SMIN_MVT_i32_ri_Predicate_simm8_32b(RetVT, Op0, imm1);
15656 default: return 0;
15657 }
15658}
15659
15660// Top-level FastEmit function.
15661
15662unsigned fastEmit_ri_Predicate_simm8_32b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15663 switch (Opcode) {
15664 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
15665 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_32b(VT, RetVT, Op0, imm1);
15666 default: return 0;
15667 }
15668}
15669
15670// FastEmit functions for ISD::SMAX.
15671
15672unsigned fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15673 if (RetVT.SimpleTy != MVT::i64)
15674 return 0;
15675 if ((Subtarget->hasCSSC())) {
15676 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15677 }
15678 return 0;
15679}
15680
15681unsigned fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15682 switch (VT.SimpleTy) {
15683 case MVT::i64: return fastEmit_ISD_SMAX_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
15684 default: return 0;
15685 }
15686}
15687
15688// FastEmit functions for ISD::SMIN.
15689
15690unsigned fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15691 if (RetVT.SimpleTy != MVT::i64)
15692 return 0;
15693 if ((Subtarget->hasCSSC())) {
15694 return fastEmitInst_ri(MachineInstOpcode: AArch64::SMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15695 }
15696 return 0;
15697}
15698
15699unsigned fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15700 switch (VT.SimpleTy) {
15701 case MVT::i64: return fastEmit_ISD_SMIN_MVT_i64_ri_Predicate_simm8_64b(RetVT, Op0, imm1);
15702 default: return 0;
15703 }
15704}
15705
15706// Top-level FastEmit function.
15707
15708unsigned fastEmit_ri_Predicate_simm8_64b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15709 switch (Opcode) {
15710 case ISD::SMAX: return fastEmit_ISD_SMAX_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
15711 case ISD::SMIN: return fastEmit_ISD_SMIN_ri_Predicate_simm8_64b(VT, RetVT, Op0, imm1);
15712 default: return 0;
15713 }
15714}
15715
15716// FastEmit functions for ISD::UMAX.
15717
15718unsigned fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15719 if (RetVT.SimpleTy != MVT::i32)
15720 return 0;
15721 if ((Subtarget->hasCSSC())) {
15722 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15723 }
15724 return 0;
15725}
15726
15727unsigned fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15728 switch (VT.SimpleTy) {
15729 case MVT::i32: return fastEmit_ISD_UMAX_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
15730 default: return 0;
15731 }
15732}
15733
15734// FastEmit functions for ISD::UMIN.
15735
15736unsigned fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15737 if (RetVT.SimpleTy != MVT::i32)
15738 return 0;
15739 if ((Subtarget->hasCSSC())) {
15740 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINWri, RC: &AArch64::GPR32RegClass, Op0, Imm: imm1);
15741 }
15742 return 0;
15743}
15744
15745unsigned fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15746 switch (VT.SimpleTy) {
15747 case MVT::i32: return fastEmit_ISD_UMIN_MVT_i32_ri_Predicate_uimm8_32b(RetVT, Op0, imm1);
15748 default: return 0;
15749 }
15750}
15751
15752// Top-level FastEmit function.
15753
15754unsigned fastEmit_ri_Predicate_uimm8_32b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15755 switch (Opcode) {
15756 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
15757 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_32b(VT, RetVT, Op0, imm1);
15758 default: return 0;
15759 }
15760}
15761
15762// FastEmit functions for ISD::UMAX.
15763
15764unsigned fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15765 if (RetVT.SimpleTy != MVT::i64)
15766 return 0;
15767 if ((Subtarget->hasCSSC())) {
15768 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMAXXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15769 }
15770 return 0;
15771}
15772
15773unsigned fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15774 switch (VT.SimpleTy) {
15775 case MVT::i64: return fastEmit_ISD_UMAX_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
15776 default: return 0;
15777 }
15778}
15779
15780// FastEmit functions for ISD::UMIN.
15781
15782unsigned fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(MVT RetVT, unsigned Op0, uint64_t imm1) {
15783 if (RetVT.SimpleTy != MVT::i64)
15784 return 0;
15785 if ((Subtarget->hasCSSC())) {
15786 return fastEmitInst_ri(MachineInstOpcode: AArch64::UMINXri, RC: &AArch64::GPR64RegClass, Op0, Imm: imm1);
15787 }
15788 return 0;
15789}
15790
15791unsigned fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Op0, uint64_t imm1) {
15792 switch (VT.SimpleTy) {
15793 case MVT::i64: return fastEmit_ISD_UMIN_MVT_i64_ri_Predicate_uimm8_64b(RetVT, Op0, imm1);
15794 default: return 0;
15795 }
15796}
15797
15798// Top-level FastEmit function.
15799
15800unsigned fastEmit_ri_Predicate_uimm8_64b(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, uint64_t imm1) {
15801 switch (Opcode) {
15802 case ISD::UMAX: return fastEmit_ISD_UMAX_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
15803 case ISD::UMIN: return fastEmit_ISD_UMIN_ri_Predicate_uimm8_64b(VT, RetVT, Op0, imm1);
15804 default: return 0;
15805 }
15806}
15807
15808// FastEmit functions for AArch64ISD::MRS.
15809
15810unsigned fastEmit_AArch64ISD_MRS_MVT_i32_i(MVT RetVT, uint64_t imm0) {
15811 if (RetVT.SimpleTy != MVT::i64)
15812 return 0;
15813 return fastEmitInst_i(MachineInstOpcode: AArch64::MRS, RC: &AArch64::GPR64RegClass, Imm: imm0);
15814}
15815
15816unsigned fastEmit_AArch64ISD_MRS_i(MVT VT, MVT RetVT, uint64_t imm0) {
15817 switch (VT.SimpleTy) {
15818 case MVT::i32: return fastEmit_AArch64ISD_MRS_MVT_i32_i(RetVT, imm0);
15819 default: return 0;
15820 }
15821}
15822
15823// FastEmit functions for ISD::Constant.
15824
15825unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
15826 if (RetVT.SimpleTy != MVT::i32)
15827 return 0;
15828 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi32imm, RC: &AArch64::GPR32RegClass, Imm: imm0);
15829}
15830
15831unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
15832 if (RetVT.SimpleTy != MVT::i64)
15833 return 0;
15834 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVi64imm, RC: &AArch64::GPR64RegClass, Imm: imm0);
15835}
15836
15837unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
15838 switch (VT.SimpleTy) {
15839 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
15840 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
15841 default: return 0;
15842 }
15843}
15844
15845// Top-level FastEmit function.
15846
15847unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
15848 if (VT == MVT::i32 && Predicate_imm0_255(Imm: imm0))
15849 if (unsigned Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
15850 return Reg;
15851
15852 if (VT == MVT::i32 && Predicate_simm6_32b(Imm: imm0))
15853 if (unsigned Reg = fastEmit_i_Predicate_simm6_32b(VT, RetVT, Opcode, imm0))
15854 return Reg;
15855
15856 switch (Opcode) {
15857 case AArch64ISD::MRS: return fastEmit_AArch64ISD_MRS_i(VT, RetVT, imm0);
15858 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
15859 default: return 0;
15860 }
15861}
15862
15863// FastEmit functions for AArch64ISD::FMOV.
15864
15865unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
15866 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
15867 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f16_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15868 }
15869 return 0;
15870}
15871
15872unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
15873 if ((Subtarget->hasFullFP16()) && (Subtarget->isNeonAvailable())) {
15874 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv8f16_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15875 }
15876 return 0;
15877}
15878
15879unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
15880 if ((Subtarget->isNeonAvailable())) {
15881 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f32_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15882 }
15883 return 0;
15884}
15885
15886unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
15887 if ((Subtarget->isNeonAvailable())) {
15888 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv4f32_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15889 }
15890 return 0;
15891}
15892
15893unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
15894 if ((Subtarget->isNeonAvailable())) {
15895 return fastEmitInst_i(MachineInstOpcode: AArch64::FMOVv2f64_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15896 }
15897 return 0;
15898}
15899
15900unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15901switch (RetVT.SimpleTy) {
15902 case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
15903 case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
15904 case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
15905 case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
15906 case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
15907 default: return 0;
15908}
15909}
15910
15911unsigned fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15912 switch (VT.SimpleTy) {
15913 case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15914 default: return 0;
15915 }
15916}
15917
15918// FastEmit functions for AArch64ISD::MOVI.
15919
15920unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
15921 if ((Subtarget->isNeonAvailable())) {
15922 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv8b_ns, RC: &AArch64::FPR64RegClass, Imm: imm0);
15923 }
15924 return 0;
15925}
15926
15927unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
15928 if ((Subtarget->isNeonAvailable())) {
15929 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv16b_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15930 }
15931 return 0;
15932}
15933
15934unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15935switch (RetVT.SimpleTy) {
15936 case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
15937 case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
15938 default: return 0;
15939}
15940}
15941
15942unsigned fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15943 switch (VT.SimpleTy) {
15944 case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15945 default: return 0;
15946 }
15947}
15948
15949// FastEmit functions for AArch64ISD::MOVIedit.
15950
15951unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
15952 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVID, RC: &AArch64::FPR64RegClass, Imm: imm0);
15953}
15954
15955unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
15956 if ((Subtarget->isNeonAvailable())) {
15957 return fastEmitInst_i(MachineInstOpcode: AArch64::MOVIv2d_ns, RC: &AArch64::FPR128RegClass, Imm: imm0);
15958 }
15959 return 0;
15960}
15961
15962unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
15963switch (RetVT.SimpleTy) {
15964 case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
15965 case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
15966 default: return 0;
15967}
15968}
15969
15970unsigned fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
15971 switch (VT.SimpleTy) {
15972 case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
15973 default: return 0;
15974 }
15975}
15976
15977// Top-level FastEmit function.
15978
15979unsigned fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
15980 switch (Opcode) {
15981 case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
15982 case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
15983 case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
15984 default: return 0;
15985 }
15986}
15987
15988// FastEmit functions for AArch64ISD::RDSVL.
15989
15990unsigned fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(MVT RetVT, uint64_t imm0) {
15991 if (RetVT.SimpleTy != MVT::i64)
15992 return 0;
15993 if ((Subtarget->hasSME())) {
15994 return fastEmitInst_i(MachineInstOpcode: AArch64::RDSVLI_XI, RC: &AArch64::GPR64RegClass, Imm: imm0);
15995 }
15996 return 0;
15997}
15998
15999unsigned fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(MVT VT, MVT RetVT, uint64_t imm0) {
16000 switch (VT.SimpleTy) {
16001 case MVT::i32: return fastEmit_AArch64ISD_RDSVL_MVT_i32_i_Predicate_simm6_32b(RetVT, imm0);
16002 default: return 0;
16003 }
16004}
16005
16006// Top-level FastEmit function.
16007
16008unsigned fastEmit_i_Predicate_simm6_32b(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
16009 switch (Opcode) {
16010 case AArch64ISD::RDSVL: return fastEmit_AArch64ISD_RDSVL_i_Predicate_simm6_32b(VT, RetVT, imm0);
16011 default: return 0;
16012 }
16013}
16014
16015